2026-02-20T21:00:16.558 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-20T21:00:16.575 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-20T21:00:16.661 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424 branch: wip-pdonnell-testing-20260219.182737-tentacle description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-journal-stress-workunit} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '61424' kernel: branch: distro kdb: true sha1: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run four rbd-mirror daemons per cluster - desc: run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon name: yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial no_nested_subset: false os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-pdonnell-testing-20260219.182737-tentacle ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: simple client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bdev async discard: true bdev enable discard: true bluestore allocator: stupid bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: f7870454149f22574086dd2ff55d050585136531 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: f7870454149f22574086dd2ff55d050585136531 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-pdonnell-testing-20260219.182737-tentacle sha1: f7870454149f22574086dd2ff55d050585136531 owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 7161 sha1: f7870454149f22574086dd2ff55d050585136531 sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: wip-pdonnell-testing-20260219.182737-tentacle suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: f7870454149f22574086dd2ff55d050585136531 targets: trial069.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEoqTcZenIAF+X4wTlKR02Zcwn/bnCYUvW9fRROi/bwifmBbXs+0+4g9t+pgqRVjFR31ZzHFEE27LW18lMALvwo= trial163.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBI41I1LStMFdfvKEyoHbBOSe5S7OwVwzjasADkZJME4I0YpV+/nK0xUPVNn+RRY2yhe50w+BDaglotviCmH4WSk= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - rbd-mirror: client: cluster1.client.mirror.0 thrash: true - rbd-mirror: client: cluster1.client.mirror.1 thrash: true - rbd-mirror: client: cluster1.client.mirror.2 thrash: true - rbd-mirror: client: cluster1.client.mirror.3 thrash: true - rbd-mirror: client: cluster2.client.mirror.0 thrash: true - rbd-mirror: client: cluster2.client.mirror.1 thrash: true - rbd-mirror: client: cluster2.client.mirror.2 thrash: true - rbd-mirror: client: cluster2.client.mirror.3 thrash: true - rbd-mirror-thrash: cluster: cluster1 - rbd-mirror-thrash: cluster: cluster2 - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_stress.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: pool RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: journal RBD_MIRROR_USE_EXISTING_CLUSTER: '1' RBD_MIRROR_USE_RBD_MIRROR: '1' timeout: 6h teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-20_16:15:01 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.1540699 2026-02-20T21:00:16.661 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa; will attempt to use it 2026-02-20T21:00:16.662 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa/tasks 2026-02-20T21:00:16.662 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-20T21:00:16.663 INFO:teuthology.task.internal:Checking packages... 2026-02-20T21:00:16.663 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'f7870454149f22574086dd2ff55d050585136531' 2026-02-20T21:00:16.663 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-20T21:00:16.663 INFO:teuthology.packaging:ref: None 2026-02-20T21:00:16.663 INFO:teuthology.packaging:tag: None 2026-02-20T21:00:16.663 INFO:teuthology.packaging:branch: wip-pdonnell-testing-20260219.182737-tentacle 2026-02-20T21:00:16.663 INFO:teuthology.packaging:sha1: f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:00:16.664 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-pdonnell-testing-20260219.182737-tentacle 2026-02-20T21:00:16.785 INFO:teuthology.task.internal:Found packages for ceph version 20.2.0-677.gf7870454 2026-02-20T21:00:16.786 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-20T21:00:16.797 INFO:teuthology.task.internal:no buildpackages task found 2026-02-20T21:00:16.797 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-20T21:00:16.811 INFO:teuthology.task.internal:Saving configuration 2026-02-20T21:00:16.821 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-20T21:00:16.832 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-20T21:00:16.892 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial069.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-20 20:56:46.427414', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEoqTcZenIAF+X4wTlKR02Zcwn/bnCYUvW9fRROi/bwifmBbXs+0+4g9t+pgqRVjFR31ZzHFEE27LW18lMALvwo='} 2026-02-20T21:00:16.947 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial163.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-20 20:56:46.428072', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBI41I1LStMFdfvKEyoHbBOSe5S7OwVwzjasADkZJME4I0YpV+/nK0xUPVNn+RRY2yhe50w+BDaglotviCmH4WSk='} 2026-02-20T21:00:16.947 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-20T21:00:16.958 INFO:teuthology.task.internal:roles: ubuntu@trial069.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-20T21:00:16.958 INFO:teuthology.task.internal:roles: ubuntu@trial163.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-20T21:00:16.958 INFO:teuthology.run_tasks:Running task console_log... 2026-02-20T21:00:17.090 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x76a3f8d13560>, signals=[15]) 2026-02-20T21:00:17.090 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-20T21:00:17.107 INFO:teuthology.task.internal:Opening connections... 2026-02-20T21:00:17.107 DEBUG:teuthology.task.internal:connecting to ubuntu@trial069.front.sepia.ceph.com 2026-02-20T21:00:17.108 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-20T21:00:17.170 DEBUG:teuthology.task.internal:connecting to ubuntu@trial163.front.sepia.ceph.com 2026-02-20T21:00:17.170 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-20T21:00:17.231 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-20T21:00:17.243 DEBUG:teuthology.orchestra.run.trial069:> uname -m 2026-02-20T21:00:17.258 INFO:teuthology.orchestra.run.trial069.stdout:x86_64 2026-02-20T21:00:17.258 DEBUG:teuthology.orchestra.run.trial069:> cat /etc/os-release 2026-02-20T21:00:17.314 INFO:teuthology.orchestra.run.trial069.stdout:NAME="CentOS Stream" 2026-02-20T21:00:17.314 INFO:teuthology.orchestra.run.trial069.stdout:VERSION="9" 2026-02-20T21:00:17.314 INFO:teuthology.orchestra.run.trial069.stdout:ID="centos" 2026-02-20T21:00:17.315 INFO:teuthology.orchestra.run.trial069.stdout:ID_LIKE="rhel fedora" 2026-02-20T21:00:17.315 INFO:teuthology.orchestra.run.trial069.stdout:VERSION_ID="9" 2026-02-20T21:00:17.315 INFO:teuthology.orchestra.run.trial069.stdout:PLATFORM_ID="platform:el9" 2026-02-20T21:00:17.315 INFO:teuthology.orchestra.run.trial069.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-20T21:00:17.315 INFO:teuthology.orchestra.run.trial069.stdout:ANSI_COLOR="0;31" 2026-02-20T21:00:17.315 INFO:teuthology.orchestra.run.trial069.stdout:LOGO="fedora-logo-icon" 2026-02-20T21:00:17.315 INFO:teuthology.orchestra.run.trial069.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-20T21:00:17.315 INFO:teuthology.orchestra.run.trial069.stdout:HOME_URL="https://centos.org/" 2026-02-20T21:00:17.315 INFO:teuthology.orchestra.run.trial069.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-20T21:00:17.315 INFO:teuthology.orchestra.run.trial069.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-20T21:00:17.315 INFO:teuthology.orchestra.run.trial069.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-20T21:00:17.316 INFO:teuthology.lock.ops:Updating trial069.front.sepia.ceph.com on lock server 2026-02-20T21:00:17.390 DEBUG:teuthology.orchestra.run.trial163:> uname -m 2026-02-20T21:00:17.406 INFO:teuthology.orchestra.run.trial163.stdout:x86_64 2026-02-20T21:00:17.406 DEBUG:teuthology.orchestra.run.trial163:> cat /etc/os-release 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:NAME="CentOS Stream" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:VERSION="9" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:ID="centos" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:ID_LIKE="rhel fedora" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:VERSION_ID="9" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:PLATFORM_ID="platform:el9" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:ANSI_COLOR="0;31" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:LOGO="fedora-logo-icon" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:HOME_URL="https://centos.org/" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-20T21:00:17.462 INFO:teuthology.orchestra.run.trial163.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-20T21:00:17.463 INFO:teuthology.lock.ops:Updating trial163.front.sepia.ceph.com on lock server 2026-02-20T21:00:17.542 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-20T21:00:17.553 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-20T21:00:17.564 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-20T21:00:17.565 DEBUG:teuthology.orchestra.run.trial069:> test '!' -e /home/ubuntu/cephtest 2026-02-20T21:00:17.566 DEBUG:teuthology.orchestra.run.trial163:> test '!' -e /home/ubuntu/cephtest 2026-02-20T21:00:17.581 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-20T21:00:17.591 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-20T21:00:17.591 DEBUG:teuthology.orchestra.run.trial069:> test -z $(ls -A /var/lib/ceph) 2026-02-20T21:00:17.622 DEBUG:teuthology.orchestra.run.trial163:> test -z $(ls -A /var/lib/ceph) 2026-02-20T21:00:17.639 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-20T21:00:17.797 INFO:teuthology.run_tasks:Running task kernel... 2026-02-20T21:00:17.819 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-20T21:00:17.819 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-20T21:00:17.819 DEBUG:teuthology.orchestra.run.trial069:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-20T21:00:17.819 DEBUG:teuthology.orchestra.run.trial163:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-20T21:00:17.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:00:17.833 DEBUG:teuthology.orchestra.run.trial069:> uname -r 2026-02-20T21:00:17.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:00:17.834 DEBUG:teuthology.orchestra.run.trial163:> uname -r 2026-02-20T21:00:17.889 INFO:teuthology.orchestra.run.trial069.stdout:5.14.0-681.el9.x86_64 2026-02-20T21:00:17.889 INFO:teuthology.task.kernel:Running kernel on trial069: 5.14.0-681.el9.x86_64 2026-02-20T21:00:17.889 DEBUG:teuthology.orchestra.run.trial069:> sudo yum install -y kernel 2026-02-20T21:00:17.890 INFO:teuthology.orchestra.run.trial163.stdout:5.14.0-681.el9.x86_64 2026-02-20T21:00:17.890 INFO:teuthology.task.kernel:Running kernel on trial163: 5.14.0-681.el9.x86_64 2026-02-20T21:00:17.890 DEBUG:teuthology.orchestra.run.trial163:> sudo yum install -y kernel 2026-02-20T21:00:18.972 INFO:teuthology.orchestra.run.trial069.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.9 MB 00:00 2026-02-20T21:00:19.182 INFO:teuthology.orchestra.run.trial163.stdout:CentOS Stream 9 - BaseOS 9.7 MB/s | 8.9 MB 00:00 2026-02-20T21:00:20.051 INFO:teuthology.orchestra.run.trial163.stdout:CentOS Stream 9 - AppStream 78 MB/s | 27 MB 00:00 2026-02-20T21:00:21.452 INFO:teuthology.orchestra.run.trial069.stdout:CentOS Stream 9 - AppStream 14 MB/s | 27 MB 00:01 2026-02-20T21:00:23.442 INFO:teuthology.orchestra.run.trial163.stdout:CentOS Stream 9 - CRB 7.4 MB/s | 7.8 MB 00:01 2026-02-20T21:00:24.293 INFO:teuthology.orchestra.run.trial163.stdout:CentOS Stream 9 - Extras packages 152 kB/s | 20 kB 00:00 2026-02-20T21:00:24.840 INFO:teuthology.orchestra.run.trial163.stdout:Extra Packages for Enterprise Linux 42 MB/s | 20 MB 00:00 2026-02-20T21:00:28.019 INFO:teuthology.orchestra.run.trial163.stdout:lab-extras 2.4 MB/s | 50 kB 00:00 2026-02-20T21:00:29.023 INFO:teuthology.orchestra.run.trial163.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-20T21:00:29.023 INFO:teuthology.orchestra.run.trial163.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-20T21:00:29.042 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:00:29.042 INFO:teuthology.orchestra.run.trial163.stdout:Nothing to do. 2026-02-20T21:00:29.043 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:00:29.067 DEBUG:teuthology.orchestra.run.trial163:> echo no | sudo yum reinstall kernel || true 2026-02-20T21:00:29.367 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:01 ago on Fri 20 Feb 2026 09:00:28 PM UTC. 2026-02-20T21:00:29.447 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:00:29.448 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:00:29.448 INFO:teuthology.orchestra.run.trial163.stdout: Package Architecture Version Repository Size 2026-02-20T21:00:29.448 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:00:29.448 INFO:teuthology.orchestra.run.trial163.stdout:Reinstalling: 2026-02-20T21:00:29.448 INFO:teuthology.orchestra.run.trial163.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-20T21:00:29.448 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:00:29.448 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:00:29.449 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:00:29.449 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:00:29.449 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 803 k 2026-02-20T21:00:29.449 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 0 2026-02-20T21:00:29.449 INFO:teuthology.orchestra.run.trial163.stderr:Operation aborted. 2026-02-20T21:00:29.468 INFO:teuthology.orchestra.run.trial163.stdout:Is this ok [y/N]: 2026-02-20T21:00:29.468 DEBUG:teuthology.orchestra.run.trial163:> sudo yum reinstall -y kernel || true 2026-02-20T21:00:29.768 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:01 ago on Fri 20 Feb 2026 09:00:28 PM UTC. 2026-02-20T21:00:29.849 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:00:29.849 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:00:29.849 INFO:teuthology.orchestra.run.trial163.stdout: Package Architecture Version Repository Size 2026-02-20T21:00:29.849 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:00:29.849 INFO:teuthology.orchestra.run.trial163.stdout:Reinstalling: 2026-02-20T21:00:29.849 INFO:teuthology.orchestra.run.trial163.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-20T21:00:29.849 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:00:29.849 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:00:29.849 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:00:29.850 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:00:29.850 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 803 k 2026-02-20T21:00:29.850 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 0 2026-02-20T21:00:29.850 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:00:30.173 INFO:teuthology.orchestra.run.trial163.stdout:kernel-5.14.0-681.el9.x86_64.rpm 12 MB/s | 803 kB 00:00 2026-02-20T21:00:30.174 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:00:30.174 INFO:teuthology.orchestra.run.trial163.stdout:Total 2.4 MB/s | 803 kB 00:00 2026-02-20T21:00:30.184 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:00:30.217 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:00:30.217 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:00:30.221 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:00:30.222 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:00:30.245 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:00:30.249 INFO:teuthology.orchestra.run.trial163.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-20T21:00:30.249 INFO:teuthology.orchestra.run.trial163.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-20T21:00:30.362 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-20T21:00:30.362 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-20T21:00:30.416 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-20T21:00:30.416 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:00:30.417 INFO:teuthology.orchestra.run.trial163.stdout:Reinstalled: 2026-02-20T21:00:30.417 INFO:teuthology.orchestra.run.trial163.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-20T21:00:30.417 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:00:30.417 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:00:30.443 DEBUG:teuthology.orchestra.run.trial163:> rpm -q kernel | sort -rV | head -n 1 2026-02-20T21:00:30.467 INFO:teuthology.orchestra.run.trial163.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-20T21:00:30.467 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-20T21:00:30.467 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-20T21:00:30.467 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-20T21:00:30.467 DEBUG:teuthology.orchestra.run.trial163:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-20T21:00:30.530 INFO:teuthology.orchestra.run.trial163.stdout:ttyS1 2026-02-20T21:00:30.563 DEBUG:teuthology.parallel:result is None 2026-02-20T21:00:32.624 INFO:teuthology.orchestra.run.trial069.stdout:CentOS Stream 9 - CRB 905 kB/s | 7.8 MB 00:08 2026-02-20T21:00:34.182 INFO:teuthology.orchestra.run.trial069.stdout:CentOS Stream 9 - Extras packages 24 kB/s | 20 kB 00:00 2026-02-20T21:00:35.020 INFO:teuthology.orchestra.run.trial069.stdout:Extra Packages for Enterprise Linux 26 MB/s | 20 MB 00:00 2026-02-20T21:00:38.259 INFO:teuthology.orchestra.run.trial069.stdout:lab-extras 2.1 MB/s | 50 kB 00:00 2026-02-20T21:00:39.272 INFO:teuthology.orchestra.run.trial069.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-20T21:00:39.272 INFO:teuthology.orchestra.run.trial069.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-20T21:00:39.291 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:00:39.292 INFO:teuthology.orchestra.run.trial069.stdout:Nothing to do. 2026-02-20T21:00:39.292 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:00:39.315 DEBUG:teuthology.orchestra.run.trial069:> echo no | sudo yum reinstall kernel || true 2026-02-20T21:00:39.656 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:00:01 ago on Fri 20 Feb 2026 09:00:38 PM UTC. 2026-02-20T21:00:39.736 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:00:39.737 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:00:39.738 INFO:teuthology.orchestra.run.trial069.stdout: Package Architecture Version Repository Size 2026-02-20T21:00:39.738 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:00:39.738 INFO:teuthology.orchestra.run.trial069.stdout:Reinstalling: 2026-02-20T21:00:39.738 INFO:teuthology.orchestra.run.trial069.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-20T21:00:39.738 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:00:39.738 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:00:39.738 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:00:39.738 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:00:39.738 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 803 k 2026-02-20T21:00:39.738 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 0 2026-02-20T21:00:39.739 INFO:teuthology.orchestra.run.trial069.stderr:Operation aborted. 2026-02-20T21:00:39.758 INFO:teuthology.orchestra.run.trial069.stdout:Is this ok [y/N]: 2026-02-20T21:00:39.758 DEBUG:teuthology.orchestra.run.trial069:> sudo yum reinstall -y kernel || true 2026-02-20T21:00:40.060 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:00:02 ago on Fri 20 Feb 2026 09:00:38 PM UTC. 2026-02-20T21:00:40.140 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:00:40.141 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:00:40.141 INFO:teuthology.orchestra.run.trial069.stdout: Package Architecture Version Repository Size 2026-02-20T21:00:40.141 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:00:40.141 INFO:teuthology.orchestra.run.trial069.stdout:Reinstalling: 2026-02-20T21:00:40.141 INFO:teuthology.orchestra.run.trial069.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-20T21:00:40.142 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:00:40.142 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:00:40.142 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:00:40.142 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:00:40.142 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 803 k 2026-02-20T21:00:40.142 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 0 2026-02-20T21:00:40.142 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:00:41.332 INFO:teuthology.orchestra.run.trial069.stdout:kernel-5.14.0-681.el9.x86_64.rpm 902 kB/s | 803 kB 00:00 2026-02-20T21:00:41.332 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:00:41.332 INFO:teuthology.orchestra.run.trial069.stdout:Total 673 kB/s | 803 kB 00:01 2026-02-20T21:00:41.342 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:00:41.376 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:00:41.376 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:00:41.380 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:00:41.381 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:00:41.404 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:00:41.408 INFO:teuthology.orchestra.run.trial069.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-20T21:00:41.408 INFO:teuthology.orchestra.run.trial069.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-20T21:00:41.524 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-20T21:00:41.524 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-20T21:00:41.580 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-20T21:00:41.580 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:00:41.581 INFO:teuthology.orchestra.run.trial069.stdout:Reinstalled: 2026-02-20T21:00:41.581 INFO:teuthology.orchestra.run.trial069.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-20T21:00:41.581 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:00:41.581 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:00:41.608 DEBUG:teuthology.orchestra.run.trial069:> rpm -q kernel | sort -rV | head -n 1 2026-02-20T21:00:41.631 INFO:teuthology.orchestra.run.trial069.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-20T21:00:41.632 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-20T21:00:41.632 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-20T21:00:41.632 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-20T21:00:41.633 DEBUG:teuthology.orchestra.run.trial069:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-20T21:00:41.696 INFO:teuthology.orchestra.run.trial069.stdout:ttyS1 2026-02-20T21:00:41.723 DEBUG:teuthology.parallel:result is None 2026-02-20T21:00:41.723 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-20T21:00:41.735 INFO:teuthology.task.internal:Creating test directory... 2026-02-20T21:00:41.735 DEBUG:teuthology.orchestra.run.trial069:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-20T21:00:41.765 DEBUG:teuthology.orchestra.run.trial163:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-20T21:00:41.782 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-20T21:00:41.794 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-20T21:00:41.810 INFO:teuthology.task.internal:Creating archive directory... 2026-02-20T21:00:41.810 DEBUG:teuthology.orchestra.run.trial069:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-20T21:00:41.821 DEBUG:teuthology.orchestra.run.trial163:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-20T21:00:41.848 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-20T21:00:41.859 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-20T21:00:41.860 DEBUG:teuthology.orchestra.run.trial069:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-20T21:00:41.878 DEBUG:teuthology.orchestra.run.trial163:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-20T21:00:41.901 INFO:teuthology.orchestra.run.trial069.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-20T21:00:41.902 INFO:teuthology.orchestra.run.trial163.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-20T21:00:41.910 INFO:teuthology.orchestra.run.trial069.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-20T21:00:41.911 INFO:teuthology.orchestra.run.trial163.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-20T21:00:41.913 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-20T21:00:41.924 INFO:teuthology.task.internal:Configuring sudo... 2026-02-20T21:00:41.924 DEBUG:teuthology.orchestra.run.trial069:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-20T21:00:41.954 DEBUG:teuthology.orchestra.run.trial163:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-20T21:00:41.978 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-20T21:00:41.993 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-20T21:00:41.993 DEBUG:teuthology.orchestra.run.trial069:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-20T21:00:42.019 DEBUG:teuthology.orchestra.run.trial163:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-20T21:00:42.037 DEBUG:teuthology.orchestra.run.trial069:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T21:00:42.170 DEBUG:teuthology.orchestra.run.trial069:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T21:00:42.195 DEBUG:teuthology.orchestra.run.trial069:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T21:00:42.331 DEBUG:teuthology.orchestra.run.trial069:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T21:00:42.356 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:00:42.356 DEBUG:teuthology.orchestra.run.trial069:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-20T21:00:42.420 DEBUG:teuthology.orchestra.run.trial163:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T21:00:42.518 DEBUG:teuthology.orchestra.run.trial163:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T21:00:42.542 DEBUG:teuthology.orchestra.run.trial163:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T21:00:42.682 DEBUG:teuthology.orchestra.run.trial163:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T21:00:42.705 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:00:42.705 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-20T21:00:42.770 DEBUG:teuthology.orchestra.run.trial069:> sudo service rsyslog restart 2026-02-20T21:00:42.772 DEBUG:teuthology.orchestra.run.trial163:> sudo service rsyslog restart 2026-02-20T21:00:42.799 INFO:teuthology.orchestra.run.trial069.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-20T21:00:42.838 INFO:teuthology.orchestra.run.trial163.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-20T21:00:43.311 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-20T21:00:43.327 INFO:teuthology.task.internal:Starting timer... 2026-02-20T21:00:43.327 INFO:teuthology.run_tasks:Running task pcp... 2026-02-20T21:00:43.356 INFO:teuthology.run_tasks:Running task selinux... 2026-02-20T21:00:43.382 DEBUG:teuthology.orchestra.run.trial069:> sudo service auditd rotate 2026-02-20T21:00:43.413 INFO:teuthology.orchestra.run.trial069.stdout:Rotating logs: 2026-02-20T21:00:43.415 DEBUG:teuthology.orchestra.run.trial163:> sudo service auditd rotate 2026-02-20T21:00:43.443 INFO:teuthology.orchestra.run.trial163.stdout:Rotating logs: 2026-02-20T21:00:43.445 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-20T21:00:43.445 DEBUG:teuthology.orchestra.run.trial069:> /usr/sbin/getenforce 2026-02-20T21:00:43.471 INFO:teuthology.orchestra.run.trial069.stdout:Permissive 2026-02-20T21:00:43.471 DEBUG:teuthology.orchestra.run.trial163:> /usr/sbin/getenforce 2026-02-20T21:00:43.501 INFO:teuthology.orchestra.run.trial163.stdout:Permissive 2026-02-20T21:00:43.501 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@trial069.front.sepia.ceph.com': 'permissive', 'ubuntu@trial163.front.sepia.ceph.com': 'permissive'} 2026-02-20T21:00:43.501 DEBUG:teuthology.orchestra.run.trial069:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2026-02-20T21:00:43.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:00:43.535 DEBUG:teuthology.orchestra.run.trial163:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2026-02-20T21:00:43.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:00:43.566 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-20T21:00:43.566 DEBUG:teuthology.orchestra.run.trial069:> sudo /usr/sbin/setenforce permissive 2026-02-20T21:00:43.602 DEBUG:teuthology.orchestra.run.trial163:> sudo /usr/sbin/setenforce permissive 2026-02-20T21:00:43.629 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-20T21:00:43.643 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2026-02-20T21:00:43.646 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2026-02-20T21:00:43.777 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-20T21:00:43.787 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-20T21:00:43.789 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial069.front.sepia.ceph.com,trial163.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-20T21:03:23.770 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial069.front.sepia.ceph.com'), Remote(name='ubuntu@trial163.front.sepia.ceph.com')] 2026-02-20T21:03:23.779 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial069.front.sepia.ceph.com' 2026-02-20T21:03:23.780 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-20T21:03:23.888 DEBUG:teuthology.orchestra.run.trial069:> true 2026-02-20T21:03:23.962 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial069.front.sepia.ceph.com' 2026-02-20T21:03:23.963 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial163.front.sepia.ceph.com' 2026-02-20T21:03:23.963 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial163.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-20T21:03:24.042 DEBUG:teuthology.orchestra.run.trial163:> true 2026-02-20T21:03:24.121 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial163.front.sepia.ceph.com' 2026-02-20T21:03:24.121 INFO:teuthology.run_tasks:Running task clock... 2026-02-20T21:03:24.132 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-20T21:03:24.132 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-20T21:03:24.133 DEBUG:teuthology.orchestra.run.trial069:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-20T21:03:24.135 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-20T21:03:24.135 DEBUG:teuthology.orchestra.run.trial163:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-20T21:03:24.160 INFO:teuthology.orchestra.run.trial069.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-20T21:03:24.172 INFO:teuthology.orchestra.run.trial069.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-20T21:03:24.189 INFO:teuthology.orchestra.run.trial163.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-20T21:03:24.191 INFO:teuthology.orchestra.run.trial069.stderr:sudo: ntpd: command not found 2026-02-20T21:03:24.200 INFO:teuthology.orchestra.run.trial163.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-20T21:03:24.201 INFO:teuthology.orchestra.run.trial069.stdout:506 Cannot talk to daemon 2026-02-20T21:03:24.212 INFO:teuthology.orchestra.run.trial069.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-20T21:03:24.221 INFO:teuthology.orchestra.run.trial163.stderr:sudo: ntpd: command not found 2026-02-20T21:03:24.223 INFO:teuthology.orchestra.run.trial069.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-20T21:03:24.231 INFO:teuthology.orchestra.run.trial163.stdout:506 Cannot talk to daemon 2026-02-20T21:03:24.242 INFO:teuthology.orchestra.run.trial163.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-20T21:03:24.253 INFO:teuthology.orchestra.run.trial163.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-20T21:03:24.266 INFO:teuthology.orchestra.run.trial069.stderr:bash: line 1: ntpq: command not found 2026-02-20T21:03:24.267 INFO:teuthology.orchestra.run.trial069.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-20T21:03:24.267 INFO:teuthology.orchestra.run.trial069.stdout:=============================================================================== 2026-02-20T21:03:24.295 INFO:teuthology.orchestra.run.trial163.stderr:bash: line 1: ntpq: command not found 2026-02-20T21:03:24.548 INFO:teuthology.orchestra.run.trial163.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-20T21:03:24.549 INFO:teuthology.orchestra.run.trial163.stdout:=============================================================================== 2026-02-20T21:03:24.549 INFO:teuthology.orchestra.run.trial163.stdout:^? ntp4.glypnod.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-20T21:03:24.549 INFO:teuthology.orchestra.run.trial163.stdout:^? 23.186.168.130 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-20T21:03:24.549 INFO:teuthology.orchestra.run.trial163.stdout:^? ntp2.zeus.frumentum.media 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-20T21:03:24.549 INFO:teuthology.orchestra.run.trial163.stdout:^? ntp6.kernfusion.at 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-20T21:03:24.550 INFO:teuthology.run_tasks:Running task install... 2026-02-20T21:03:24.567 DEBUG:teuthology.task.install:project ceph 2026-02-20T21:03:24.567 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}} 2026-02-20T21:03:24.567 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'} 2026-02-20T21:03:24.567 INFO:teuthology.task.install:Using flavor: default 2026-02-20T21:03:24.576 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-20T21:03:24.576 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-20T21:03:24.576 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f7870454149f22574086dd2ff55d050585136531', 'tag': None, 'wait_for_package': False} 2026-02-20T21:03:24.577 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:03:24.579 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f7870454149f22574086dd2ff55d050585136531', 'tag': None, 'wait_for_package': False} 2026-02-20T21:03:24.580 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:03:24.769 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/ 2026-02-20T21:03:24.769 INFO:teuthology.task.install.rpm:Package version is 20.2.0-677.gf7870454 2026-02-20T21:03:24.779 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/ 2026-02-20T21:03:24.779 INFO:teuthology.task.install.rpm:Package version is 20.2.0-677.gf7870454 2026-02-20T21:03:24.938 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-20T21:03:24.938 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:03:24.938 DEBUG:teuthology.orchestra.run.trial069:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-20T21:03:24.947 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/wip-pdonnell-testing-20260219.182737-tentacle/f7870454149f22574086dd2ff55d050585136531/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-20T21:03:24.948 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:03:24.948 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-20T21:03:24.963 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-20T21:03:24.963 DEBUG:teuthology.orchestra.run.trial069:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f7870454149f22574086dd2ff55d050585136531/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-20T21:03:24.973 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-20T21:03:24.973 DEBUG:teuthology.orchestra.run.trial163:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f7870454149f22574086dd2ff55d050585136531/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-20T21:03:25.027 DEBUG:teuthology.orchestra.run.trial069:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-20T21:03:25.036 DEBUG:teuthology.orchestra.run.trial163:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-20T21:03:25.099 DEBUG:teuthology.orchestra.run.trial069:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-20T21:03:25.111 DEBUG:teuthology.orchestra.run.trial163:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-20T21:03:25.163 INFO:teuthology.orchestra.run.trial069.stdout:check_obsoletes = 1 2026-02-20T21:03:25.164 DEBUG:teuthology.orchestra.run.trial069:> sudo yum clean all 2026-02-20T21:03:25.175 INFO:teuthology.orchestra.run.trial163.stdout:check_obsoletes = 1 2026-02-20T21:03:25.177 DEBUG:teuthology.orchestra.run.trial163:> sudo yum clean all 2026-02-20T21:03:25.324 INFO:teuthology.orchestra.run.trial069.stdout:41 files removed 2026-02-20T21:03:25.333 INFO:teuthology.orchestra.run.trial163.stdout:41 files removed 2026-02-20T21:03:25.339 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install ceph-radosgw 2026-02-20T21:03:25.348 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install ceph-radosgw 2026-02-20T21:03:25.621 INFO:teuthology.orchestra.run.trial069.stdout:ceph packages for x86_64 635 kB/s | 87 kB 00:00 2026-02-20T21:03:25.632 INFO:teuthology.orchestra.run.trial163.stdout:ceph packages for x86_64 616 kB/s | 87 kB 00:00 2026-02-20T21:03:25.911 INFO:teuthology.orchestra.run.trial069.stdout:ceph noarch packages 68 kB/s | 18 kB 00:00 2026-02-20T21:03:25.930 INFO:teuthology.orchestra.run.trial163.stdout:ceph noarch packages 66 kB/s | 18 kB 00:00 2026-02-20T21:03:26.087 INFO:teuthology.orchestra.run.trial069.stdout:ceph source packages 12 kB/s | 1.9 kB 00:00 2026-02-20T21:03:26.093 INFO:teuthology.orchestra.run.trial163.stdout:ceph source packages 14 kB/s | 1.9 kB 00:00 2026-02-20T21:03:26.935 INFO:teuthology.orchestra.run.trial163.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.9 MB 00:00 2026-02-20T21:03:27.085 INFO:teuthology.orchestra.run.trial069.stdout:CentOS Stream 9 - BaseOS 9.1 MB/s | 8.9 MB 00:00 2026-02-20T21:03:30.188 INFO:teuthology.orchestra.run.trial069.stdout:CentOS Stream 9 - AppStream 10 MB/s | 27 MB 00:02 2026-02-20T21:03:32.833 INFO:teuthology.orchestra.run.trial069.stdout:CentOS Stream 9 - CRB 31 MB/s | 7.8 MB 00:00 2026-02-20T21:03:34.673 INFO:teuthology.orchestra.run.trial069.stdout:CentOS Stream 9 - Extras packages 18 kB/s | 20 kB 00:01 2026-02-20T21:03:35.514 INFO:teuthology.orchestra.run.trial069.stdout:Extra Packages for Enterprise Linux 26 MB/s | 20 MB 00:00 2026-02-20T21:03:35.638 INFO:teuthology.orchestra.run.trial163.stdout:CentOS Stream 9 - AppStream 3.3 MB/s | 27 MB 00:08 2026-02-20T21:03:38.315 INFO:teuthology.orchestra.run.trial163.stdout:CentOS Stream 9 - CRB 26 MB/s | 7.8 MB 00:00 2026-02-20T21:03:38.672 INFO:teuthology.orchestra.run.trial069.stdout:lab-extras 2.7 MB/s | 50 kB 00:00 2026-02-20T21:03:39.869 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:03:39.870 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:03:39.870 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repository Size 2026-02-20T21:03:39.870 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:03:39.870 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:03:39.870 INFO:teuthology.orchestra.run.trial069.stdout: ceph-radosgw x86_64 2:20.2.0-677.gf7870454.el9 ceph 24 M 2026-02-20T21:03:39.871 INFO:teuthology.orchestra.run.trial069.stdout:Upgrading: 2026-02-20T21:03:39.871 INFO:teuthology.orchestra.run.trial069.stdout: librados2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 3.5 M 2026-02-20T21:03:39.871 INFO:teuthology.orchestra.run.trial069.stdout: librbd1 x86_64 2:20.2.0-677.gf7870454.el9 ceph 2.8 M 2026-02-20T21:03:39.871 INFO:teuthology.orchestra.run.trial069.stdout:Installing dependencies: 2026-02-20T21:03:39.871 INFO:teuthology.orchestra.run.trial069.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-20T21:03:39.871 INFO:teuthology.orchestra.run.trial069.stdout: ceph-base x86_64 2:20.2.0-677.gf7870454.el9 ceph 5.9 M 2026-02-20T21:03:39.871 INFO:teuthology.orchestra.run.trial069.stdout: ceph-common x86_64 2:20.2.0-677.gf7870454.el9 ceph 24 M 2026-02-20T21:03:39.871 INFO:teuthology.orchestra.run.trial069.stdout: ceph-selinux x86_64 2:20.2.0-677.gf7870454.el9 ceph 25 k 2026-02-20T21:03:39.871 INFO:teuthology.orchestra.run.trial069.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-20T21:03:39.871 INFO:teuthology.orchestra.run.trial069.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-20T21:03:39.871 INFO:teuthology.orchestra.run.trial069.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-20T21:03:39.872 INFO:teuthology.orchestra.run.trial069.stdout: libcephfs2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 869 k 2026-02-20T21:03:39.872 INFO:teuthology.orchestra.run.trial069.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-20T21:03:39.872 INFO:teuthology.orchestra.run.trial069.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-20T21:03:39.872 INFO:teuthology.orchestra.run.trial069.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-20T21:03:39.872 INFO:teuthology.orchestra.run.trial069.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-20T21:03:39.872 INFO:teuthology.orchestra.run.trial069.stdout: libradosstriper1 x86_64 2:20.2.0-677.gf7870454.el9 ceph 250 k 2026-02-20T21:03:39.872 INFO:teuthology.orchestra.run.trial069.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-20T21:03:39.872 INFO:teuthology.orchestra.run.trial069.stdout: librgw2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 6.4 M 2026-02-20T21:03:39.872 INFO:teuthology.orchestra.run.trial069.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-20T21:03:39.872 INFO:teuthology.orchestra.run.trial069.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-20T21:03:39.873 INFO:teuthology.orchestra.run.trial069.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-20T21:03:39.873 INFO:teuthology.orchestra.run.trial069.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-20T21:03:39.873 INFO:teuthology.orchestra.run.trial069.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-20T21:03:39.873 INFO:teuthology.orchestra.run.trial069.stdout: python3-ceph-argparse x86_64 2:20.2.0-677.gf7870454.el9 ceph 45 k 2026-02-20T21:03:39.873 INFO:teuthology.orchestra.run.trial069.stdout: python3-ceph-common x86_64 2:20.2.0-677.gf7870454.el9 ceph 175 k 2026-02-20T21:03:39.873 INFO:teuthology.orchestra.run.trial069.stdout: python3-cephfs x86_64 2:20.2.0-677.gf7870454.el9 ceph 163 k 2026-02-20T21:03:39.873 INFO:teuthology.orchestra.run.trial069.stdout: python3-rados x86_64 2:20.2.0-677.gf7870454.el9 ceph 323 k 2026-02-20T21:03:39.873 INFO:teuthology.orchestra.run.trial069.stdout: python3-rbd x86_64 2:20.2.0-677.gf7870454.el9 ceph 305 k 2026-02-20T21:03:39.873 INFO:teuthology.orchestra.run.trial069.stdout: python3-rgw x86_64 2:20.2.0-677.gf7870454.el9 ceph 99 k 2026-02-20T21:03:39.873 INFO:teuthology.orchestra.run.trial069.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-20T21:03:39.874 INFO:teuthology.orchestra.run.trial069.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-20T21:03:39.874 INFO:teuthology.orchestra.run.trial069.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-20T21:03:39.874 INFO:teuthology.orchestra.run.trial069.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-20T21:03:39.874 INFO:teuthology.orchestra.run.trial069.stdout:Installing weak dependencies: 2026-02-20T21:03:39.874 INFO:teuthology.orchestra.run.trial069.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-20T21:03:39.874 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:03:39.874 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:03:39.874 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:03:39.874 INFO:teuthology.orchestra.run.trial069.stdout:Install 32 Packages 2026-02-20T21:03:39.874 INFO:teuthology.orchestra.run.trial069.stdout:Upgrade 2 Packages 2026-02-20T21:03:39.875 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:03:39.875 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 78 M 2026-02-20T21:03:39.875 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:03:40.342 INFO:teuthology.orchestra.run.trial163.stdout:CentOS Stream 9 - Extras packages 15 kB/s | 20 kB 00:01 2026-02-20T21:03:40.446 INFO:teuthology.orchestra.run.trial069.stdout:(1/34): ceph-base-20.2.0-677.gf7870454.el9.x86_ 16 MB/s | 5.9 MB 00:00 2026-02-20T21:03:40.479 INFO:teuthology.orchestra.run.trial069.stdout:(2/34): ceph-selinux-20.2.0-677.gf7870454.el9.x 770 kB/s | 25 kB 00:00 2026-02-20T21:03:40.531 INFO:teuthology.orchestra.run.trial069.stdout:(3/34): libcephfs2-20.2.0-677.gf7870454.el9.x86 16 MB/s | 869 kB 00:00 2026-02-20T21:03:40.572 INFO:teuthology.orchestra.run.trial069.stdout:(4/34): libradosstriper1-20.2.0-677.gf7870454.e 6.0 MB/s | 250 kB 00:00 2026-02-20T21:03:40.946 INFO:teuthology.orchestra.run.trial069.stdout:(5/34): librgw2-20.2.0-677.gf7870454.el9.x86_64 17 MB/s | 6.4 MB 00:00 2026-02-20T21:03:40.997 INFO:teuthology.orchestra.run.trial069.stdout:(6/34): python3-ceph-argparse-20.2.0-677.gf7870 889 kB/s | 45 kB 00:00 2026-02-20T21:03:40.998 INFO:teuthology.orchestra.run.trial163.stdout:Extra Packages for Enterprise Linux 34 MB/s | 20 MB 00:00 2026-02-20T21:03:41.050 INFO:teuthology.orchestra.run.trial069.stdout:(7/34): python3-ceph-common-20.2.0-677.gf787045 3.2 MB/s | 175 kB 00:00 2026-02-20T21:03:41.099 INFO:teuthology.orchestra.run.trial069.stdout:(8/34): python3-cephfs-20.2.0-677.gf7870454.el9 3.3 MB/s | 163 kB 00:00 2026-02-20T21:03:41.153 INFO:teuthology.orchestra.run.trial069.stdout:(9/34): python3-rados-20.2.0-677.gf7870454.el9. 5.8 MB/s | 323 kB 00:00 2026-02-20T21:03:41.206 INFO:teuthology.orchestra.run.trial069.stdout:(10/34): ceph-radosgw-20.2.0-677.gf7870454.el9. 21 MB/s | 24 MB 00:01 2026-02-20T21:03:41.208 INFO:teuthology.orchestra.run.trial069.stdout:(11/34): python3-rbd-20.2.0-677.gf7870454.el9.x 5.4 MB/s | 305 kB 00:00 2026-02-20T21:03:41.251 INFO:teuthology.orchestra.run.trial069.stdout:(12/34): ceph-common-20.2.0-677.gf7870454.el9.x 20 MB/s | 24 MB 00:01 2026-02-20T21:03:41.252 INFO:teuthology.orchestra.run.trial069.stdout:(13/34): python3-rgw-20.2.0-677.gf7870454.el9.x 2.1 MB/s | 99 kB 00:00 2026-02-20T21:03:41.292 INFO:teuthology.orchestra.run.trial069.stdout:(14/34): boost-program-options-1.75.0-13.el9.x8 2.5 MB/s | 104 kB 00:00 2026-02-20T21:03:41.293 INFO:teuthology.orchestra.run.trial069.stdout:(15/34): mailcap-2.1.49-5.el9.noarch.rpm 390 kB/s | 33 kB 00:00 2026-02-20T21:03:41.310 INFO:teuthology.orchestra.run.trial069.stdout:(16/34): libnbd-1.20.3-4.el9.x86_64.rpm 9.1 MB/s | 164 kB 00:00 2026-02-20T21:03:41.315 INFO:teuthology.orchestra.run.trial069.stdout:(17/34): pciutils-3.7.0-7.el9.x86_64.rpm 1.4 MB/s | 93 kB 00:00 2026-02-20T21:03:41.320 INFO:teuthology.orchestra.run.trial069.stdout:(18/34): librabbitmq-0.11.0-7.el9.x86_64.rpm 4.7 MB/s | 45 kB 00:00 2026-02-20T21:03:41.336 INFO:teuthology.orchestra.run.trial069.stdout:(19/34): libpmemobj-1.12.1-1.el9.x86_64.rpm 3.7 MB/s | 160 kB 00:00 2026-02-20T21:03:41.337 INFO:teuthology.orchestra.run.trial069.stdout:(20/34): lttng-ust-2.12.0-6.el9.x86_64.rpm 16 MB/s | 292 kB 00:00 2026-02-20T21:03:41.346 INFO:teuthology.orchestra.run.trial069.stdout:(21/34): qatlib-service-25.08.0-2.el9.x86_64.rp 4.1 MB/s | 37 kB 00:00 2026-02-20T21:03:41.354 INFO:teuthology.orchestra.run.trial069.stdout:(22/34): qatlib-25.08.0-2.el9.x86_64.rpm 13 MB/s | 240 kB 00:00 2026-02-20T21:03:41.356 INFO:teuthology.orchestra.run.trial069.stdout:(23/34): qatzip-libs-1.3.1-1.el9.x86_64.rpm 6.9 MB/s | 66 kB 00:00 2026-02-20T21:03:41.374 INFO:teuthology.orchestra.run.trial069.stdout:(24/34): librdkafka-1.6.1-102.el9.x86_64.rpm 11 MB/s | 662 kB 00:00 2026-02-20T21:03:41.573 INFO:teuthology.orchestra.run.trial069.stdout:(25/34): libarrow-doc-9.0.0-15.el9.noarch.rpm 124 kB/s | 25 kB 00:00 2026-02-20T21:03:41.708 INFO:teuthology.orchestra.run.trial069.stdout:(26/34): gperftools-libs-2.9.1-3.el9.x86_64.rpm 871 kB/s | 308 kB 00:00 2026-02-20T21:03:41.708 INFO:teuthology.orchestra.run.trial069.stdout:(27/34): liboath-2.6.12-1.el9.x86_64.rpm 364 kB/s | 49 kB 00:00 2026-02-20T21:03:41.729 INFO:teuthology.orchestra.run.trial069.stdout:(28/34): libunwind-1.6.2-1.el9.x86_64.rpm 3.2 MB/s | 67 kB 00:00 2026-02-20T21:03:41.892 INFO:teuthology.orchestra.run.trial069.stdout:(29/34): parquet-libs-9.0.0-15.el9.x86_64.rpm 4.5 MB/s | 838 kB 00:00 2026-02-20T21:03:41.893 INFO:teuthology.orchestra.run.trial069.stdout:(30/34): re2-20211101-20.el9.x86_64.rpm 1.1 MB/s | 191 kB 00:00 2026-02-20T21:03:41.965 INFO:teuthology.orchestra.run.trial069.stdout:(31/34): librados2-20.2.0-677.gf7870454.el9.x86 49 MB/s | 3.5 MB 00:00 2026-02-20T21:03:42.019 INFO:teuthology.orchestra.run.trial069.stdout:(32/34): librbd1-20.2.0-677.gf7870454.el9.x86_6 53 MB/s | 2.8 MB 00:00 2026-02-20T21:03:42.441 INFO:teuthology.orchestra.run.trial069.stdout:(33/34): thrift-0.15.0-4.el9.x86_64.rpm 2.9 MB/s | 1.6 MB 00:00 2026-02-20T21:03:44.192 INFO:teuthology.orchestra.run.trial163.stdout:lab-extras 2.6 MB/s | 50 kB 00:00 2026-02-20T21:03:45.384 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:03:45.385 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:03:45.385 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repository Size 2026-02-20T21:03:45.385 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:03:45.385 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:03:45.385 INFO:teuthology.orchestra.run.trial163.stdout: ceph-radosgw x86_64 2:20.2.0-677.gf7870454.el9 ceph 24 M 2026-02-20T21:03:45.385 INFO:teuthology.orchestra.run.trial163.stdout:Upgrading: 2026-02-20T21:03:45.386 INFO:teuthology.orchestra.run.trial163.stdout: librados2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 3.5 M 2026-02-20T21:03:45.386 INFO:teuthology.orchestra.run.trial163.stdout: librbd1 x86_64 2:20.2.0-677.gf7870454.el9 ceph 2.8 M 2026-02-20T21:03:45.386 INFO:teuthology.orchestra.run.trial163.stdout:Installing dependencies: 2026-02-20T21:03:45.386 INFO:teuthology.orchestra.run.trial163.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-20T21:03:45.386 INFO:teuthology.orchestra.run.trial163.stdout: ceph-base x86_64 2:20.2.0-677.gf7870454.el9 ceph 5.9 M 2026-02-20T21:03:45.386 INFO:teuthology.orchestra.run.trial163.stdout: ceph-common x86_64 2:20.2.0-677.gf7870454.el9 ceph 24 M 2026-02-20T21:03:45.386 INFO:teuthology.orchestra.run.trial163.stdout: ceph-selinux x86_64 2:20.2.0-677.gf7870454.el9 ceph 25 k 2026-02-20T21:03:45.386 INFO:teuthology.orchestra.run.trial163.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-20T21:03:45.386 INFO:teuthology.orchestra.run.trial163.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-20T21:03:45.386 INFO:teuthology.orchestra.run.trial163.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-20T21:03:45.386 INFO:teuthology.orchestra.run.trial163.stdout: libcephfs2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 869 k 2026-02-20T21:03:45.386 INFO:teuthology.orchestra.run.trial163.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: libradosstriper1 x86_64 2:20.2.0-677.gf7870454.el9 ceph 250 k 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: librgw2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 6.4 M 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: python3-ceph-argparse x86_64 2:20.2.0-677.gf7870454.el9 ceph 45 k 2026-02-20T21:03:45.387 INFO:teuthology.orchestra.run.trial163.stdout: python3-ceph-common x86_64 2:20.2.0-677.gf7870454.el9 ceph 175 k 2026-02-20T21:03:45.388 INFO:teuthology.orchestra.run.trial163.stdout: python3-cephfs x86_64 2:20.2.0-677.gf7870454.el9 ceph 163 k 2026-02-20T21:03:45.388 INFO:teuthology.orchestra.run.trial163.stdout: python3-rados x86_64 2:20.2.0-677.gf7870454.el9 ceph 323 k 2026-02-20T21:03:45.388 INFO:teuthology.orchestra.run.trial163.stdout: python3-rbd x86_64 2:20.2.0-677.gf7870454.el9 ceph 305 k 2026-02-20T21:03:45.388 INFO:teuthology.orchestra.run.trial163.stdout: python3-rgw x86_64 2:20.2.0-677.gf7870454.el9 ceph 99 k 2026-02-20T21:03:45.388 INFO:teuthology.orchestra.run.trial163.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-20T21:03:45.388 INFO:teuthology.orchestra.run.trial163.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-20T21:03:45.388 INFO:teuthology.orchestra.run.trial163.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-20T21:03:45.388 INFO:teuthology.orchestra.run.trial163.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-20T21:03:45.388 INFO:teuthology.orchestra.run.trial163.stdout:Installing weak dependencies: 2026-02-20T21:03:45.388 INFO:teuthology.orchestra.run.trial163.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-20T21:03:45.388 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:03:45.388 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:03:45.389 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:03:45.389 INFO:teuthology.orchestra.run.trial163.stdout:Install 32 Packages 2026-02-20T21:03:45.389 INFO:teuthology.orchestra.run.trial163.stdout:Upgrade 2 Packages 2026-02-20T21:03:45.389 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:03:45.389 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 78 M 2026-02-20T21:03:45.390 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:03:46.412 INFO:teuthology.orchestra.run.trial163.stdout:(1/34): ceph-base-20.2.0-677.gf7870454.el9.x86_ 9.8 MB/s | 5.9 MB 00:00 2026-02-20T21:03:46.449 INFO:teuthology.orchestra.run.trial163.stdout:(2/34): ceph-selinux-20.2.0-677.gf7870454.el9.x 675 kB/s | 25 kB 00:00 2026-02-20T21:03:46.555 INFO:teuthology.orchestra.run.trial163.stdout:(3/34): libcephfs2-20.2.0-677.gf7870454.el9.x86 8.0 MB/s | 869 kB 00:00 2026-02-20T21:03:46.609 INFO:teuthology.orchestra.run.trial163.stdout:(4/34): libradosstriper1-20.2.0-677.gf7870454.e 4.6 MB/s | 250 kB 00:00 2026-02-20T21:03:47.163 INFO:teuthology.orchestra.run.trial163.stdout:(5/34): ceph-common-20.2.0-677.gf7870454.el9.x8 18 MB/s | 24 MB 00:01 2026-02-20T21:03:47.203 INFO:teuthology.orchestra.run.trial163.stdout:(6/34): python3-ceph-argparse-20.2.0-677.gf7870 1.1 MB/s | 45 kB 00:00 2026-02-20T21:03:47.218 INFO:teuthology.orchestra.run.trial163.stdout:(7/34): librgw2-20.2.0-677.gf7870454.el9.x86_64 10 MB/s | 6.4 MB 00:00 2026-02-20T21:03:47.239 INFO:teuthology.orchestra.run.trial163.stdout:(8/34): python3-ceph-common-20.2.0-677.gf787045 4.7 MB/s | 175 kB 00:00 2026-02-20T21:03:47.251 INFO:teuthology.orchestra.run.trial163.stdout:(9/34): python3-cephfs-20.2.0-677.gf7870454.el9 4.8 MB/s | 163 kB 00:00 2026-02-20T21:03:47.280 INFO:teuthology.orchestra.run.trial163.stdout:(10/34): python3-rados-20.2.0-677.gf7870454.el9 7.8 MB/s | 323 kB 00:00 2026-02-20T21:03:47.297 INFO:teuthology.orchestra.run.trial163.stdout:(11/34): python3-rbd-20.2.0-677.gf7870454.el9.x 6.6 MB/s | 305 kB 00:00 2026-02-20T21:03:47.320 INFO:teuthology.orchestra.run.trial163.stdout:(12/34): python3-rgw-20.2.0-677.gf7870454.el9.x 2.4 MB/s | 99 kB 00:00 2026-02-20T21:03:47.521 INFO:teuthology.orchestra.run.trial163.stdout:(13/34): ceph-radosgw-20.2.0-677.gf7870454.el9. 14 MB/s | 24 MB 00:01 2026-02-20T21:03:47.548 INFO:teuthology.orchestra.run.trial163.stdout:(14/34): boost-program-options-1.75.0-13.el9.x8 3.8 MB/s | 104 kB 00:00 2026-02-20T21:03:47.557 INFO:teuthology.orchestra.run.trial163.stdout:(15/34): libnbd-1.20.3-4.el9.x86_64.rpm 18 MB/s | 164 kB 00:00 2026-02-20T21:03:47.564 INFO:teuthology.orchestra.run.trial163.stdout:(16/34): libpmemobj-1.12.1-1.el9.x86_64.rpm 23 MB/s | 160 kB 00:00 2026-02-20T21:03:47.568 INFO:teuthology.orchestra.run.trial163.stdout:(17/34): librabbitmq-0.11.0-7.el9.x86_64.rpm 10 MB/s | 45 kB 00:00 2026-02-20T21:03:47.579 INFO:teuthology.orchestra.run.trial163.stdout:(18/34): librdkafka-1.6.1-102.el9.x86_64.rpm 63 MB/s | 662 kB 00:00 2026-02-20T21:03:47.585 INFO:teuthology.orchestra.run.trial163.stdout:(19/34): lttng-ust-2.12.0-6.el9.x86_64.rpm 45 MB/s | 292 kB 00:00 2026-02-20T21:03:47.592 INFO:teuthology.orchestra.run.trial163.stdout:(20/34): qatlib-25.08.0-2.el9.x86_64.rpm 38 MB/s | 240 kB 00:00 2026-02-20T21:03:47.596 INFO:teuthology.orchestra.run.trial163.stdout:(21/34): qatlib-service-25.08.0-2.el9.x86_64.rp 8.5 MB/s | 37 kB 00:00 2026-02-20T21:03:47.601 INFO:teuthology.orchestra.run.trial163.stdout:(22/34): qatzip-libs-1.3.1-1.el9.x86_64.rpm 14 MB/s | 66 kB 00:00 2026-02-20T21:03:47.747 INFO:teuthology.orchestra.run.trial163.stdout:(23/34): gperftools-libs-2.9.1-3.el9.x86_64.rpm 2.1 MB/s | 308 kB 00:00 2026-02-20T21:03:47.851 INFO:teuthology.orchestra.run.trial163.stdout:(24/34): libarrow-9.0.0-15.el9.x86_64.rpm 42 MB/s | 4.4 MB 00:00 2026-02-20T21:03:47.853 INFO:teuthology.orchestra.run.trial163.stdout:(25/34): mailcap-2.1.49-5.el9.noarch.rpm 60 kB/s | 33 kB 00:00 2026-02-20T21:03:47.876 INFO:teuthology.orchestra.run.trial163.stdout:(26/34): libarrow-doc-9.0.0-15.el9.noarch.rpm 973 kB/s | 25 kB 00:00 2026-02-20T21:03:47.901 INFO:teuthology.orchestra.run.trial163.stdout:(27/34): libunwind-1.6.2-1.el9.x86_64.rpm 2.7 MB/s | 67 kB 00:00 2026-02-20T21:03:47.931 INFO:teuthology.orchestra.run.trial163.stdout:(28/34): parquet-libs-9.0.0-15.el9.x86_64.rpm 27 MB/s | 838 kB 00:00 2026-02-20T21:03:47.945 INFO:teuthology.orchestra.run.trial163.stdout:(29/34): liboath-2.6.12-1.el9.x86_64.rpm 533 kB/s | 49 kB 00:00 2026-02-20T21:03:47.954 INFO:teuthology.orchestra.run.trial163.stdout:(30/34): pciutils-3.7.0-7.el9.x86_64.rpm 147 kB/s | 93 kB 00:00 2026-02-20T21:03:47.960 INFO:teuthology.orchestra.run.trial163.stdout:(31/34): re2-20211101-20.el9.x86_64.rpm 6.6 MB/s | 191 kB 00:00 2026-02-20T21:03:48.054 INFO:teuthology.orchestra.run.trial163.stdout:(32/34): librbd1-20.2.0-677.gf7870454.el9.x86_6 30 MB/s | 2.8 MB 00:00 2026-02-20T21:03:48.061 INFO:teuthology.orchestra.run.trial163.stdout:(33/34): thrift-0.15.0-4.el9.x86_64.rpm 14 MB/s | 1.6 MB 00:00 2026-02-20T21:03:48.101 INFO:teuthology.orchestra.run.trial163.stdout:(34/34): librados2-20.2.0-677.gf7870454.el9.x86 24 MB/s | 3.5 MB 00:00 2026-02-20T21:03:48.103 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:03:48.103 INFO:teuthology.orchestra.run.trial163.stdout:Total 29 MB/s | 78 MB 00:02 2026-02-20T21:03:48.193 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:03:48.210 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:03:48.210 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:03:48.506 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:03:48.506 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:03:48.836 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:03:48.849 INFO:teuthology.orchestra.run.trial163.stdout: Installing : thrift-0.15.0-4.el9.x86_64 1/36 2026-02-20T21:03:49.003 INFO:teuthology.orchestra.run.trial163.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 2/36 2026-02-20T21:03:49.008 INFO:teuthology.orchestra.run.trial163.stdout: Upgrading : librados2-2:20.2.0-677.gf7870454.el9.x86_64 3/36 2026-02-20T21:03:49.036 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: librados2-2:20.2.0-677.gf7870454.el9.x86_64 3/36 2026-02-20T21:03:49.073 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 4/36 2026-02-20T21:03:49.074 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 5/36 2026-02-20T21:03:49.084 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 5/36 2026-02-20T21:03:49.092 INFO:teuthology.orchestra.run.trial163.stdout: Installing : liboath-2.6.12-1.el9.x86_64 6/36 2026-02-20T21:03:49.094 INFO:teuthology.orchestra.run.trial163.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 7/36 2026-02-20T21:03:49.097 INFO:teuthology.orchestra.run.trial163.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 8/36 2026-02-20T21:03:49.102 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 9/36 2026-02-20T21:03:49.110 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-ceph-argparse-2:20.2.0-677.gf7870454.el9.x 10/36 2026-02-20T21:03:49.122 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 11/36 2026-02-20T21:03:49.123 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 12/36 2026-02-20T21:03:49.136 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 12/36 2026-02-20T21:03:49.167 INFO:teuthology.orchestra.run.trial163.stdout: Installing : re2-1:20211101-20.el9.x86_64 13/36 2026-02-20T21:03:49.173 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 14/36 2026-02-20T21:03:49.180 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 15/36 2026-02-20T21:03:49.182 INFO:teuthology.orchestra.run.trial163.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 16/36 2026-02-20T21:03:49.187 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 17/36 2026-02-20T21:03:49.310 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/36 2026-02-20T21:03:49.311 INFO:teuthology.orchestra.run.trial163.stdout: Upgrading : librbd1-2:20.2.0-677.gf7870454.el9.x86_64 19/36 2026-02-20T21:03:49.337 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: librbd1-2:20.2.0-677.gf7870454.el9.x86_64 19/36 2026-02-20T21:03:49.340 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 20/36 2026-02-20T21:03:49.346 INFO:teuthology.orchestra.run.trial163.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/36 2026-02-20T21:03:49.622 INFO:teuthology.orchestra.run.trial163.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/36 2026-02-20T21:03:49.623 INFO:teuthology.orchestra.run.trial163.stdout: Installing : librgw2-2:20.2.0-677.gf7870454.el9.x86_64 23/36 2026-02-20T21:03:49.640 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: librgw2-2:20.2.0-677.gf7870454.el9.x86_64 23/36 2026-02-20T21:03:49.647 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 24/36 2026-02-20T21:03:49.654 INFO:teuthology.orchestra.run.trial163.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/36 2026-02-20T21:03:49.657 INFO:teuthology.orchestra.run.trial163.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/36 2026-02-20T21:03:49.658 INFO:teuthology.orchestra.run.trial163.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/36 2026-02-20T21:03:49.675 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/36 2026-02-20T21:03:49.678 INFO:teuthology.orchestra.run.trial163.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/36 2026-02-20T21:03:49.710 INFO:teuthology.orchestra.run.trial163.stdout: Installing : mailcap-2.1.49-5.el9.noarch 29/36 2026-02-20T21:03:49.712 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-ceph-common-2:20.2.0-677.gf7870454.el9.x86 30/36 2026-02-20T21:03:50.782 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 31/36 2026-02-20T21:03:50.784 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 31/36 2026-02-20T21:03:51.078 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 31/36 2026-02-20T21:03:51.081 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 32/36 2026-02-20T21:03:51.121 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 32/36 2026-02-20T21:03:51.121 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-20T21:03:51.121 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-20T21:03:51.121 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:03:51.125 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 33/36 2026-02-20T21:03:55.608 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 33/36 2026-02-20T21:03:55.608 INFO:teuthology.orchestra.run.trial163.stdout:skipping the directory /sys 2026-02-20T21:03:55.608 INFO:teuthology.orchestra.run.trial163.stdout:skipping the directory /proc 2026-02-20T21:03:55.609 INFO:teuthology.orchestra.run.trial163.stdout:skipping the directory /mnt 2026-02-20T21:03:55.609 INFO:teuthology.orchestra.run.trial163.stdout:skipping the directory /var/tmp 2026-02-20T21:03:55.609 INFO:teuthology.orchestra.run.trial163.stdout:skipping the directory /home 2026-02-20T21:03:55.609 INFO:teuthology.orchestra.run.trial163.stdout:skipping the directory /root 2026-02-20T21:03:55.609 INFO:teuthology.orchestra.run.trial163.stdout:skipping the directory /tmp 2026-02-20T21:03:55.609 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:03:56.637 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 34/36 2026-02-20T21:03:56.661 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 34/36 2026-02-20T21:03:56.661 INFO:teuthology.orchestra.run.trial163.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:03:56.661 INFO:teuthology.orchestra.run.trial163.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-20T21:03:56.661 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-20T21:03:56.662 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-20T21:03:56.662 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:03:56.663 INFO:teuthology.orchestra.run.trial163.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 35/36 2026-02-20T21:03:56.675 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 35/36 2026-02-20T21:03:56.675 INFO:teuthology.orchestra.run.trial163.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 36/36 2026-02-20T21:03:56.731 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 36/36 2026-02-20T21:03:56.731 INFO:teuthology.orchestra.run.trial163.stdout:Creating group 'qat' with GID 992. 2026-02-20T21:03:56.732 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:03:57.088 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 1/36 2026-02-20T21:03:57.088 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 2/36 2026-02-20T21:03:57.088 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 3/36 2026-02-20T21:03:57.088 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 4/36 2026-02-20T21:03:57.089 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 5/36 2026-02-20T21:03:57.089 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 6/36 2026-02-20T21:03:57.089 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : librgw2-2:20.2.0-677.gf7870454.el9.x86_64 7/36 2026-02-20T21:03:57.089 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-ceph-argparse-2:20.2.0-677.gf7870454.el9.x 8/36 2026-02-20T21:03:57.089 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-ceph-common-2:20.2.0-677.gf7870454.el9.x86 9/36 2026-02-20T21:03:57.089 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 10/36 2026-02-20T21:03:57.089 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 11/36 2026-02-20T21:03:57.089 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 12/36 2026-02-20T21:03:57.089 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 13/36 2026-02-20T21:03:57.089 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/36 2026-02-20T21:03:57.089 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 15/36 2026-02-20T21:03:57.089 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 16/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 17/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 18/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 19/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 20/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 21/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 22/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 23/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 24/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 25/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 26/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 27/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 28/36 2026-02-20T21:03:57.090 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 29/36 2026-02-20T21:03:57.091 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 30/36 2026-02-20T21:03:57.091 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : re2-1:20211101-20.el9.x86_64 31/36 2026-02-20T21:03:57.091 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 32/36 2026-02-20T21:03:57.091 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : librados2-2:20.2.0-677.gf7870454.el9.x86_64 33/36 2026-02-20T21:03:57.091 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 34/36 2026-02-20T21:03:57.091 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : librbd1-2:20.2.0-677.gf7870454.el9.x86_64 35/36 2026-02-20T21:03:57.148 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 36/36 2026-02-20T21:03:57.148 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:03:57.148 INFO:teuthology.orchestra.run.trial163.stdout:Upgraded: 2026-02-20T21:03:57.148 INFO:teuthology.orchestra.run.trial163.stdout: librados2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.148 INFO:teuthology.orchestra.run.trial163.stdout: librbd1-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.148 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:03:57.148 INFO:teuthology.orchestra.run.trial163.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-20T21:03:57.148 INFO:teuthology.orchestra.run.trial163.stdout: ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.149 INFO:teuthology.orchestra.run.trial163.stdout: ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.149 INFO:teuthology.orchestra.run.trial163.stdout: ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.149 INFO:teuthology.orchestra.run.trial163.stdout: ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.149 INFO:teuthology.orchestra.run.trial163.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-20T21:03:57.149 INFO:teuthology.orchestra.run.trial163.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-20T21:03:57.149 INFO:teuthology.orchestra.run.trial163.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-20T21:03:57.149 INFO:teuthology.orchestra.run.trial163.stdout: libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.149 INFO:teuthology.orchestra.run.trial163.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-20T21:03:57.149 INFO:teuthology.orchestra.run.trial163.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-20T21:03:57.149 INFO:teuthology.orchestra.run.trial163.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-20T21:03:57.149 INFO:teuthology.orchestra.run.trial163.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-20T21:03:57.149 INFO:teuthology.orchestra.run.trial163.stdout: libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.150 INFO:teuthology.orchestra.run.trial163.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-20T21:03:57.150 INFO:teuthology.orchestra.run.trial163.stdout: librgw2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.150 INFO:teuthology.orchestra.run.trial163.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-20T21:03:57.150 INFO:teuthology.orchestra.run.trial163.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-20T21:03:57.150 INFO:teuthology.orchestra.run.trial163.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-20T21:03:57.150 INFO:teuthology.orchestra.run.trial163.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-20T21:03:57.150 INFO:teuthology.orchestra.run.trial163.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-20T21:03:57.150 INFO:teuthology.orchestra.run.trial163.stdout: python3-ceph-argparse-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.150 INFO:teuthology.orchestra.run.trial163.stdout: python3-ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.150 INFO:teuthology.orchestra.run.trial163.stdout: python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.151 INFO:teuthology.orchestra.run.trial163.stdout: python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.151 INFO:teuthology.orchestra.run.trial163.stdout: python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.151 INFO:teuthology.orchestra.run.trial163.stdout: python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:03:57.151 INFO:teuthology.orchestra.run.trial163.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-20T21:03:57.151 INFO:teuthology.orchestra.run.trial163.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-20T21:03:57.151 INFO:teuthology.orchestra.run.trial163.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-20T21:03:57.151 INFO:teuthology.orchestra.run.trial163.stdout: re2-1:20211101-20.el9.x86_64 2026-02-20T21:03:57.151 INFO:teuthology.orchestra.run.trial163.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-20T21:03:57.151 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:03:57.151 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:03:57.191 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install ceph-test 2026-02-20T21:03:57.502 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:13 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:03:57.584 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:03:57.585 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:03:57.585 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repository Size 2026-02-20T21:03:57.585 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:03:57.585 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:03:57.585 INFO:teuthology.orchestra.run.trial163.stdout: ceph-test x86_64 2:20.2.0-677.gf7870454.el9 ceph 84 M 2026-02-20T21:03:57.585 INFO:teuthology.orchestra.run.trial163.stdout:Installing dependencies: 2026-02-20T21:03:57.585 INFO:teuthology.orchestra.run.trial163.stdout: libcephsqlite x86_64 2:20.2.0-677.gf7870454.el9 ceph 164 k 2026-02-20T21:03:57.585 INFO:teuthology.orchestra.run.trial163.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-20T21:03:57.585 INFO:teuthology.orchestra.run.trial163.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-20T21:03:57.586 INFO:teuthology.orchestra.run.trial163.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-20T21:03:57.586 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:03:57.586 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:03:57.586 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:03:57.586 INFO:teuthology.orchestra.run.trial163.stdout:Install 5 Packages 2026-02-20T21:03:57.586 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:03:57.586 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 85 M 2026-02-20T21:03:57.586 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 365 M 2026-02-20T21:03:57.586 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:03:57.787 INFO:teuthology.orchestra.run.trial163.stdout:(1/5): libcephsqlite-20.2.0-677.gf7870454.el9.x 1.4 MB/s | 164 kB 00:00 2026-02-20T21:03:58.439 INFO:teuthology.orchestra.run.trial163.stdout:(2/5): libxslt-1.1.34-12.el9.x86_64.rpm 304 kB/s | 233 kB 00:00 2026-02-20T21:03:58.589 INFO:teuthology.orchestra.run.trial163.stdout:(3/5): socat-1.7.4.1-8.el9.x86_64.rpm 378 kB/s | 303 kB 00:00 2026-02-20T21:03:58.618 INFO:teuthology.orchestra.run.trial163.stdout:(4/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 357 kB/s | 64 kB 00:00 2026-02-20T21:03:59.431 INFO:teuthology.orchestra.run.trial163.stdout:(5/5): ceph-test-20.2.0-677.gf7870454.el9.x86_6 48 MB/s | 84 MB 00:01 2026-02-20T21:03:59.433 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:03:59.433 INFO:teuthology.orchestra.run.trial163.stdout:Total 46 MB/s | 85 MB 00:01 2026-02-20T21:03:59.458 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:03:59.468 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:03:59.468 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:03:59.755 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:03:59.755 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:00.069 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:00.075 INFO:teuthology.orchestra.run.trial163.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-20T21:04:00.079 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-20T21:04:00.086 INFO:teuthology.orchestra.run.trial163.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-20T21:04:00.087 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 4/5 2026-02-20T21:04:00.914 INFO:teuthology.orchestra.run.trial069.stdout:(34/34): libarrow-9.0.0-15.el9.x86_64.rpm 231 kB/s | 4.4 MB 00:19 2026-02-20T21:04:00.917 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:00.917 INFO:teuthology.orchestra.run.trial069.stdout:Total 3.7 MB/s | 78 MB 00:21 2026-02-20T21:04:01.008 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:01.026 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:01.027 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:01.322 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:01.322 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:01.651 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:01.665 INFO:teuthology.orchestra.run.trial069.stdout: Installing : thrift-0.15.0-4.el9.x86_64 1/36 2026-02-20T21:04:01.818 INFO:teuthology.orchestra.run.trial069.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 2/36 2026-02-20T21:04:01.822 INFO:teuthology.orchestra.run.trial069.stdout: Upgrading : librados2-2:20.2.0-677.gf7870454.el9.x86_64 3/36 2026-02-20T21:04:01.851 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: librados2-2:20.2.0-677.gf7870454.el9.x86_64 3/36 2026-02-20T21:04:01.889 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 4/36 2026-02-20T21:04:01.890 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 5/36 2026-02-20T21:04:01.902 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 5/36 2026-02-20T21:04:01.910 INFO:teuthology.orchestra.run.trial069.stdout: Installing : liboath-2.6.12-1.el9.x86_64 6/36 2026-02-20T21:04:01.913 INFO:teuthology.orchestra.run.trial069.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 7/36 2026-02-20T21:04:01.916 INFO:teuthology.orchestra.run.trial069.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 8/36 2026-02-20T21:04:01.921 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 9/36 2026-02-20T21:04:01.929 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-ceph-argparse-2:20.2.0-677.gf7870454.el9.x 10/36 2026-02-20T21:04:01.941 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 11/36 2026-02-20T21:04:01.942 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 12/36 2026-02-20T21:04:01.955 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 12/36 2026-02-20T21:04:01.991 INFO:teuthology.orchestra.run.trial069.stdout: Installing : re2-1:20211101-20.el9.x86_64 13/36 2026-02-20T21:04:01.996 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 14/36 2026-02-20T21:04:02.002 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 15/36 2026-02-20T21:04:02.005 INFO:teuthology.orchestra.run.trial069.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 16/36 2026-02-20T21:04:02.009 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 17/36 2026-02-20T21:04:02.132 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/36 2026-02-20T21:04:02.133 INFO:teuthology.orchestra.run.trial069.stdout: Upgrading : librbd1-2:20.2.0-677.gf7870454.el9.x86_64 19/36 2026-02-20T21:04:02.159 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: librbd1-2:20.2.0-677.gf7870454.el9.x86_64 19/36 2026-02-20T21:04:02.161 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 20/36 2026-02-20T21:04:02.167 INFO:teuthology.orchestra.run.trial069.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/36 2026-02-20T21:04:02.445 INFO:teuthology.orchestra.run.trial069.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/36 2026-02-20T21:04:02.446 INFO:teuthology.orchestra.run.trial069.stdout: Installing : librgw2-2:20.2.0-677.gf7870454.el9.x86_64 23/36 2026-02-20T21:04:02.463 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: librgw2-2:20.2.0-677.gf7870454.el9.x86_64 23/36 2026-02-20T21:04:02.469 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 24/36 2026-02-20T21:04:02.473 INFO:teuthology.orchestra.run.trial069.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/36 2026-02-20T21:04:02.476 INFO:teuthology.orchestra.run.trial069.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/36 2026-02-20T21:04:02.477 INFO:teuthology.orchestra.run.trial069.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/36 2026-02-20T21:04:02.494 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/36 2026-02-20T21:04:02.498 INFO:teuthology.orchestra.run.trial069.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/36 2026-02-20T21:04:02.530 INFO:teuthology.orchestra.run.trial069.stdout: Installing : mailcap-2.1.49-5.el9.noarch 29/36 2026-02-20T21:04:02.532 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-ceph-common-2:20.2.0-677.gf7870454.el9.x86 30/36 2026-02-20T21:04:03.603 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 31/36 2026-02-20T21:04:03.606 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 31/36 2026-02-20T21:04:03.798 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 4/5 2026-02-20T21:04:03.816 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 5/5 2026-02-20T21:04:03.907 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 31/36 2026-02-20T21:04:03.910 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 32/36 2026-02-20T21:04:03.951 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 32/36 2026-02-20T21:04:03.951 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-20T21:04:03.951 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-20T21:04:03.951 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:03.955 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 33/36 2026-02-20T21:04:04.120 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 5/5 2026-02-20T21:04:04.121 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 1/5 2026-02-20T21:04:04.121 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 2/5 2026-02-20T21:04:04.121 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-20T21:04:04.121 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-20T21:04:04.172 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-20T21:04:04.173 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:04.173 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:04.173 INFO:teuthology.orchestra.run.trial163.stdout: ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:04.173 INFO:teuthology.orchestra.run.trial163.stdout: libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:04.173 INFO:teuthology.orchestra.run.trial163.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-20T21:04:04.173 INFO:teuthology.orchestra.run.trial163.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-20T21:04:04.173 INFO:teuthology.orchestra.run.trial163.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-20T21:04:04.173 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:04.173 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:04.208 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install ceph 2026-02-20T21:04:04.513 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:20 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:04.594 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:04.596 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:04.596 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repository Size 2026-02-20T21:04:04.596 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:04.596 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:04.596 INFO:teuthology.orchestra.run.trial163.stdout: ceph x86_64 2:20.2.0-677.gf7870454.el9 ceph 6.5 k 2026-02-20T21:04:04.596 INFO:teuthology.orchestra.run.trial163.stdout:Installing dependencies: 2026-02-20T21:04:04.596 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mds x86_64 2:20.2.0-677.gf7870454.el9 ceph 2.3 M 2026-02-20T21:04:04.596 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr x86_64 2:20.2.0-677.gf7870454.el9 ceph 961 k 2026-02-20T21:04:04.597 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-modules-core noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 290 k 2026-02-20T21:04:04.597 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mon x86_64 2:20.2.0-677.gf7870454.el9 ceph 5.0 M 2026-02-20T21:04:04.597 INFO:teuthology.orchestra.run.trial163.stdout: ceph-osd x86_64 2:20.2.0-677.gf7870454.el9 ceph 17 M 2026-02-20T21:04:04.597 INFO:teuthology.orchestra.run.trial163.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-20T21:04:04.597 INFO:teuthology.orchestra.run.trial163.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-20T21:04:04.597 INFO:teuthology.orchestra.run.trial163.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-20T21:04:04.597 INFO:teuthology.orchestra.run.trial163.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-20T21:04:04.597 INFO:teuthology.orchestra.run.trial163.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-20T21:04:04.597 INFO:teuthology.orchestra.run.trial163.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-20T21:04:04.597 INFO:teuthology.orchestra.run.trial163.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-20T21:04:04.597 INFO:teuthology.orchestra.run.trial163.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-20T21:04:04.597 INFO:teuthology.orchestra.run.trial163.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-20T21:04:04.598 INFO:teuthology.orchestra.run.trial163.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-20T21:04:04.598 INFO:teuthology.orchestra.run.trial163.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-20T21:04:04.598 INFO:teuthology.orchestra.run.trial163.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-20T21:04:04.598 INFO:teuthology.orchestra.run.trial163.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-20T21:04:04.598 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-20T21:04:04.598 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-20T21:04:04.598 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-20T21:04:04.598 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-20T21:04:04.598 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-20T21:04:04.598 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-20T21:04:04.598 INFO:teuthology.orchestra.run.trial163.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-20T21:04:04.598 INFO:teuthology.orchestra.run.trial163.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-20T21:04:04.599 INFO:teuthology.orchestra.run.trial163.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-20T21:04:04.599 INFO:teuthology.orchestra.run.trial163.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-20T21:04:04.599 INFO:teuthology.orchestra.run.trial163.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-20T21:04:04.599 INFO:teuthology.orchestra.run.trial163.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-20T21:04:04.599 INFO:teuthology.orchestra.run.trial163.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-20T21:04:04.599 INFO:teuthology.orchestra.run.trial163.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-20T21:04:04.599 INFO:teuthology.orchestra.run.trial163.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-20T21:04:04.599 INFO:teuthology.orchestra.run.trial163.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-20T21:04:04.599 INFO:teuthology.orchestra.run.trial163.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-20T21:04:04.599 INFO:teuthology.orchestra.run.trial163.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-20T21:04:04.599 INFO:teuthology.orchestra.run.trial163.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-20T21:04:04.599 INFO:teuthology.orchestra.run.trial163.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-20T21:04:04.600 INFO:teuthology.orchestra.run.trial163.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-20T21:04:04.600 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:04.600 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:04.600 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:04.600 INFO:teuthology.orchestra.run.trial163.stdout:Install 40 Packages 2026-02-20T21:04:04.600 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:04.601 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 30 M 2026-02-20T21:04:04.601 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 100 M 2026-02-20T21:04:04.601 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:04.908 INFO:teuthology.orchestra.run.trial163.stdout:(1/40): ceph-20.2.0-677.gf7870454.el9.x86_64.rp 126 kB/s | 6.5 kB 00:00 2026-02-20T21:04:05.002 INFO:teuthology.orchestra.run.trial163.stdout:(2/40): ceph-mgr-20.2.0-677.gf7870454.el9.x86_6 6.4 MB/s | 961 kB 00:00 2026-02-20T21:04:05.025 INFO:teuthology.orchestra.run.trial163.stdout:(3/40): ceph-mds-20.2.0-677.gf7870454.el9.x86_6 14 MB/s | 2.3 MB 00:00 2026-02-20T21:04:05.064 INFO:teuthology.orchestra.run.trial163.stdout:(4/40): ceph-mgr-modules-core-20.2.0-677.gf7870 7.3 MB/s | 290 kB 00:00 2026-02-20T21:04:05.097 INFO:teuthology.orchestra.run.trial163.stdout:(5/40): ledmon-libs-1.1.0-3.el9.x86_64.rpm 1.2 MB/s | 40 kB 00:00 2026-02-20T21:04:05.120 INFO:teuthology.orchestra.run.trial163.stdout:(6/40): libconfig-1.7.2-9.el9.x86_64.rpm 3.2 MB/s | 72 kB 00:00 2026-02-20T21:04:05.146 INFO:teuthology.orchestra.run.trial163.stdout:(7/40): python3-cffi-1.14.5-5.el9.x86_64.rpm 9.4 MB/s | 253 kB 00:00 2026-02-20T21:04:05.185 INFO:teuthology.orchestra.run.trial163.stdout:(8/40): python3-cryptography-36.0.1-5.el9.x86_6 32 MB/s | 1.2 MB 00:00 2026-02-20T21:04:05.197 INFO:teuthology.orchestra.run.trial163.stdout:(9/40): python3-ply-3.11-14.el9.noarch.rpm 8.9 MB/s | 106 kB 00:00 2026-02-20T21:04:05.209 INFO:teuthology.orchestra.run.trial163.stdout:(10/40): python3-pycparser-2.20-6.el9.noarch.rp 11 MB/s | 135 kB 00:00 2026-02-20T21:04:05.223 INFO:teuthology.orchestra.run.trial163.stdout:(11/40): ceph-mon-20.2.0-677.gf7870454.el9.x86_ 16 MB/s | 5.0 MB 00:00 2026-02-20T21:04:05.224 INFO:teuthology.orchestra.run.trial163.stdout:(12/40): python3-pyparsing-2.4.7-9.el9.noarch.r 9.9 MB/s | 150 kB 00:00 2026-02-20T21:04:05.238 INFO:teuthology.orchestra.run.trial163.stdout:(13/40): python3-urllib3-1.26.5-7.el9.noarch.rp 15 MB/s | 218 kB 00:00 2026-02-20T21:04:05.250 INFO:teuthology.orchestra.run.trial163.stdout:(14/40): unzip-6.0-59.el9.x86_64.rpm 15 MB/s | 182 kB 00:00 2026-02-20T21:04:05.265 INFO:teuthology.orchestra.run.trial163.stdout:(15/40): zip-3.0-35.el9.x86_64.rpm 18 MB/s | 266 kB 00:00 2026-02-20T21:04:05.272 INFO:teuthology.orchestra.run.trial163.stdout:(16/40): python3-requests-2.25.1-10.el9.noarch. 2.5 MB/s | 126 kB 00:00 2026-02-20T21:04:05.470 INFO:teuthology.orchestra.run.trial163.stdout:(17/40): ceph-osd-20.2.0-677.gf7870454.el9.x86_ 36 MB/s | 17 MB 00:00 2026-02-20T21:04:06.150 INFO:teuthology.orchestra.run.trial163.stdout:(18/40): lua-5.4.4-4.el9.x86_64.rpm 215 kB/s | 188 kB 00:00 2026-02-20T21:04:06.230 INFO:teuthology.orchestra.run.trial163.stdout:(19/40): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 255 kB/s | 246 kB 00:00 2026-02-20T21:04:06.264 INFO:teuthology.orchestra.run.trial163.stdout:(20/40): python3-libstoragemgmt-1.10.1-1.el9.x8 223 kB/s | 177 kB 00:00 2026-02-20T21:04:06.388 INFO:teuthology.orchestra.run.trial163.stdout:(21/40): python3-packaging-20.9-5.el9.noarch.rp 324 kB/s | 77 kB 00:00 2026-02-20T21:04:06.389 INFO:teuthology.orchestra.run.trial163.stdout:(22/40): python3-toml-0.10.2-6.el9.noarch.rpm 262 kB/s | 42 kB 00:00 2026-02-20T21:04:06.430 INFO:teuthology.orchestra.run.trial163.stdout:(23/40): python3-autocommand-2.2.2-8.el9.noarch 729 kB/s | 29 kB 00:00 2026-02-20T21:04:06.454 INFO:teuthology.orchestra.run.trial163.stdout:(24/40): luarocks-3.9.2-5.el9.noarch.rpm 2.2 MB/s | 151 kB 00:00 2026-02-20T21:04:06.458 INFO:teuthology.orchestra.run.trial163.stdout:(25/40): python3-backports-tarfile-1.2.0-1.el9. 2.1 MB/s | 60 kB 00:00 2026-02-20T21:04:06.464 INFO:teuthology.orchestra.run.trial163.stdout:(26/40): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 4.1 MB/s | 43 kB 00:00 2026-02-20T21:04:06.472 INFO:teuthology.orchestra.run.trial163.stdout:(27/40): python3-cheroot-10.0.1-4.el9.noarch.rp 12 MB/s | 173 kB 00:00 2026-02-20T21:04:06.484 INFO:teuthology.orchestra.run.trial163.stdout:(28/40): python3-cherrypy-18.6.1-2.el9.noarch.r 19 MB/s | 358 kB 00:00 2026-02-20T21:04:06.484 INFO:teuthology.orchestra.run.trial163.stdout:(29/40): python3-jaraco-8.2.1-3.el9.noarch.rpm 923 kB/s | 11 kB 00:00 2026-02-20T21:04:06.493 INFO:teuthology.orchestra.run.trial163.stdout:(30/40): python3-jaraco-classes-3.2.1-5.el9.noa 1.9 MB/s | 18 kB 00:00 2026-02-20T21:04:06.494 INFO:teuthology.orchestra.run.trial163.stdout:(31/40): python3-jaraco-collections-3.0.0-8.el9 2.3 MB/s | 23 kB 00:00 2026-02-20T21:04:06.502 INFO:teuthology.orchestra.run.trial163.stdout:(32/40): python3-jaraco-context-6.0.1-3.el9.noa 2.1 MB/s | 20 kB 00:00 2026-02-20T21:04:06.503 INFO:teuthology.orchestra.run.trial163.stdout:(33/40): python3-jaraco-functools-3.5.0-2.el9.n 2.1 MB/s | 19 kB 00:00 2026-02-20T21:04:06.506 INFO:teuthology.orchestra.run.trial163.stdout:(34/40): lua-devel-5.4.4-4.el9.x86_64.rpm 92 kB/s | 22 kB 00:00 2026-02-20T21:04:06.512 INFO:teuthology.orchestra.run.trial163.stdout:(35/40): python3-jaraco-text-4.0.0-2.el9.noarch 2.8 MB/s | 26 kB 00:00 2026-02-20T21:04:06.513 INFO:teuthology.orchestra.run.trial163.stdout:(36/40): python3-more-itertools-8.12.0-2.el9.no 7.9 MB/s | 79 kB 00:00 2026-02-20T21:04:06.515 INFO:teuthology.orchestra.run.trial163.stdout:(37/40): python3-portend-3.1.0-2.el9.noarch.rpm 1.8 MB/s | 16 kB 00:00 2026-02-20T21:04:06.522 INFO:teuthology.orchestra.run.trial163.stdout:(38/40): python3-pyOpenSSL-21.0.0-1.el9.noarch. 9.1 MB/s | 90 kB 00:00 2026-02-20T21:04:06.522 INFO:teuthology.orchestra.run.trial163.stdout:(39/40): python3-tempora-5.0.0-2.el9.noarch.rpm 3.7 MB/s | 36 kB 00:00 2026-02-20T21:04:06.525 INFO:teuthology.orchestra.run.trial163.stdout:(40/40): python3-zc-lockfile-2.0-10.el9.noarch. 2.0 MB/s | 20 kB 00:00 2026-02-20T21:04:06.527 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:06.528 INFO:teuthology.orchestra.run.trial163.stdout:Total 16 MB/s | 30 MB 00:01 2026-02-20T21:04:06.668 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:06.686 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:06.686 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:06.829 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:06.830 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:07.020 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:07.022 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/40 2026-02-20T21:04:07.027 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/40 2026-02-20T21:04:07.032 INFO:teuthology.orchestra.run.trial163.stdout: Installing : lua-5.4.4-4.el9.x86_64 3/40 2026-02-20T21:04:07.034 INFO:teuthology.orchestra.run.trial163.stdout: Installing : unzip-6.0-59.el9.x86_64 4/40 2026-02-20T21:04:07.039 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 5/40 2026-02-20T21:04:07.063 INFO:teuthology.orchestra.run.trial163.stdout: Installing : zip-3.0-35.el9.x86_64 6/40 2026-02-20T21:04:07.068 INFO:teuthology.orchestra.run.trial163.stdout: Installing : luarocks-3.9.2-5.el9.noarch 7/40 2026-02-20T21:04:07.073 INFO:teuthology.orchestra.run.trial163.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 8/40 2026-02-20T21:04:07.077 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 9/40 2026-02-20T21:04:07.083 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 10/40 2026-02-20T21:04:07.087 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 11/40 2026-02-20T21:04:07.093 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 12/40 2026-02-20T21:04:07.098 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 13/40 2026-02-20T21:04:07.101 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 14/40 2026-02-20T21:04:07.107 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 15/40 2026-02-20T21:04:07.112 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 16/40 2026-02-20T21:04:07.117 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 17/40 2026-02-20T21:04:07.120 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 18/40 2026-02-20T21:04:07.143 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 19/40 2026-02-20T21:04:07.155 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 20/40 2026-02-20T21:04:07.160 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 21/40 2026-02-20T21:04:07.170 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 22/40 2026-02-20T21:04:07.176 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-packaging-20.9-5.el9.noarch 23/40 2026-02-20T21:04:07.189 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-ply-3.11-14.el9.noarch 24/40 2026-02-20T21:04:07.204 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 25/40 2026-02-20T21:04:07.278 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 26/40 2026-02-20T21:04:07.286 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 27/40 2026-02-20T21:04:07.309 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 28/40 2026-02-20T21:04:07.338 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 29/40 2026-02-20T21:04:07.343 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 30/40 2026-02-20T21:04:07.389 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 31/40 2026-02-20T21:04:07.432 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-mgr-modules-core-2:20.2.0-677.gf7870454.el9.n 32/40 2026-02-20T21:04:07.433 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 33/40 2026-02-20T21:04:07.455 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 33/40 2026-02-20T21:04:07.455 INFO:teuthology.orchestra.run.trial163.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:07.455 INFO:teuthology.orchestra.run.trial163.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-20T21:04:07.455 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-20T21:04:07.456 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-20T21:04:07.456 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:07.460 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 34/40 2026-02-20T21:04:07.477 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 35/40 2026-02-20T21:04:07.477 INFO:teuthology.orchestra.run.trial163.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-20T21:04:07.477 INFO:teuthology.orchestra.run.trial163.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-20T21:04:07.477 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:07.486 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 35/40 2026-02-20T21:04:07.510 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 35/40 2026-02-20T21:04:07.510 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-20T21:04:07.510 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:08.193 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 36/40 2026-02-20T21:04:08.195 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 37/40 2026-02-20T21:04:08.220 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 37/40 2026-02-20T21:04:08.220 INFO:teuthology.orchestra.run.trial163.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:08.220 INFO:teuthology.orchestra.run.trial163.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-20T21:04:08.220 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-20T21:04:08.220 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-20T21:04:08.220 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:08.410 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 33/36 2026-02-20T21:04:08.410 INFO:teuthology.orchestra.run.trial069.stdout:skipping the directory /sys 2026-02-20T21:04:08.410 INFO:teuthology.orchestra.run.trial069.stdout:skipping the directory /proc 2026-02-20T21:04:08.411 INFO:teuthology.orchestra.run.trial069.stdout:skipping the directory /mnt 2026-02-20T21:04:08.411 INFO:teuthology.orchestra.run.trial069.stdout:skipping the directory /var/tmp 2026-02-20T21:04:08.411 INFO:teuthology.orchestra.run.trial069.stdout:skipping the directory /home 2026-02-20T21:04:08.411 INFO:teuthology.orchestra.run.trial069.stdout:skipping the directory /root 2026-02-20T21:04:08.411 INFO:teuthology.orchestra.run.trial069.stdout:skipping the directory /tmp 2026-02-20T21:04:08.411 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:08.433 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 38/40 2026-02-20T21:04:08.454 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 38/40 2026-02-20T21:04:08.455 INFO:teuthology.orchestra.run.trial163.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:08.455 INFO:teuthology.orchestra.run.trial163.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-20T21:04:08.455 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-20T21:04:08.455 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-20T21:04:08.455 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:08.551 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 39/40 2026-02-20T21:04:08.571 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 39/40 2026-02-20T21:04:08.571 INFO:teuthology.orchestra.run.trial163.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:08.571 INFO:teuthology.orchestra.run.trial163.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-20T21:04:08.571 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-20T21:04:08.571 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-20T21:04:08.571 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:08.588 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-2:20.2.0-677.gf7870454.el9.x86_64 40/40 2026-02-20T21:04:08.905 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-2:20.2.0-677.gf7870454.el9.x86_64 40/40 2026-02-20T21:04:08.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-2:20.2.0-677.gf7870454.el9.x86_64 1/40 2026-02-20T21:04:08.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 2/40 2026-02-20T21:04:08.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 3/40 2026-02-20T21:04:08.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 4/40 2026-02-20T21:04:08.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 5/40 2026-02-20T21:04:08.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-677.gf7870454.el9.n 6/40 2026-02-20T21:04:08.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 7/40 2026-02-20T21:04:08.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 8/40 2026-02-20T21:04:08.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 9/40 2026-02-20T21:04:08.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 10/40 2026-02-20T21:04:08.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-ply-3.11-14.el9.noarch 11/40 2026-02-20T21:04:08.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 12/40 2026-02-20T21:04:08.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 13/40 2026-02-20T21:04:08.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 14/40 2026-02-20T21:04:08.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 15/40 2026-02-20T21:04:08.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : unzip-6.0-59.el9.x86_64 16/40 2026-02-20T21:04:08.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : zip-3.0-35.el9.x86_64 17/40 2026-02-20T21:04:08.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 18/40 2026-02-20T21:04:08.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : lua-5.4.4-4.el9.x86_64 19/40 2026-02-20T21:04:08.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 20/40 2026-02-20T21:04:08.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 21/40 2026-02-20T21:04:08.908 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 22/40 2026-02-20T21:04:08.908 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 23/40 2026-02-20T21:04:08.908 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 24/40 2026-02-20T21:04:08.908 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 25/40 2026-02-20T21:04:08.908 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 26/40 2026-02-20T21:04:08.908 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 27/40 2026-02-20T21:04:08.908 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 28/40 2026-02-20T21:04:08.908 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 29/40 2026-02-20T21:04:08.908 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 30/40 2026-02-20T21:04:08.908 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 31/40 2026-02-20T21:04:08.908 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 32/40 2026-02-20T21:04:08.908 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 33/40 2026-02-20T21:04:08.909 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 34/40 2026-02-20T21:04:08.909 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 35/40 2026-02-20T21:04:08.909 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 36/40 2026-02-20T21:04:08.909 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 37/40 2026-02-20T21:04:08.909 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 38/40 2026-02-20T21:04:08.909 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 39/40 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 40/40 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: ceph-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-modules-core-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: lua-5.4.4-4.el9.x86_64 2026-02-20T21:04:08.961 INFO:teuthology.orchestra.run.trial163.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-20T21:04:08.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-ply-3.11-14.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: unzip-6.0-59.el9.x86_64 2026-02-20T21:04:08.963 INFO:teuthology.orchestra.run.trial163.stdout: zip-3.0-35.el9.x86_64 2026-02-20T21:04:08.964 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:08.964 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:08.998 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install ceph-base 2026-02-20T21:04:09.306 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:25 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:09.370 INFO:teuthology.orchestra.run.trial163.stdout:Package ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:09.387 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:09.387 INFO:teuthology.orchestra.run.trial163.stdout:Nothing to do. 2026-02-20T21:04:09.387 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:09.408 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install cephadm 2026-02-20T21:04:09.438 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 34/36 2026-02-20T21:04:09.462 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 34/36 2026-02-20T21:04:09.463 INFO:teuthology.orchestra.run.trial069.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:09.463 INFO:teuthology.orchestra.run.trial069.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-20T21:04:09.463 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-20T21:04:09.463 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-20T21:04:09.463 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:09.464 INFO:teuthology.orchestra.run.trial069.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 35/36 2026-02-20T21:04:09.477 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 35/36 2026-02-20T21:04:09.477 INFO:teuthology.orchestra.run.trial069.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 36/36 2026-02-20T21:04:09.533 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 36/36 2026-02-20T21:04:09.533 INFO:teuthology.orchestra.run.trial069.stdout:Creating group 'qat' with GID 992. 2026-02-20T21:04:09.533 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:09.707 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:25 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:09.789 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:09.790 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:09.790 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repository Size 2026-02-20T21:04:09.790 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:09.790 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:09.790 INFO:teuthology.orchestra.run.trial163.stdout: cephadm noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 1.0 M 2026-02-20T21:04:09.790 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:09.790 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:09.790 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:09.791 INFO:teuthology.orchestra.run.trial163.stdout:Install 1 Package 2026-02-20T21:04:09.791 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:09.791 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 1.0 M 2026-02-20T21:04:09.791 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 1.0 M 2026-02-20T21:04:09.791 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:09.882 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 1/36 2026-02-20T21:04:09.883 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 2/36 2026-02-20T21:04:09.883 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 3/36 2026-02-20T21:04:09.883 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 4/36 2026-02-20T21:04:09.883 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 5/36 2026-02-20T21:04:09.883 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 6/36 2026-02-20T21:04:09.883 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : librgw2-2:20.2.0-677.gf7870454.el9.x86_64 7/36 2026-02-20T21:04:09.883 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-ceph-argparse-2:20.2.0-677.gf7870454.el9.x 8/36 2026-02-20T21:04:09.883 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-ceph-common-2:20.2.0-677.gf7870454.el9.x86 9/36 2026-02-20T21:04:09.883 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 10/36 2026-02-20T21:04:09.883 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 11/36 2026-02-20T21:04:09.884 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 12/36 2026-02-20T21:04:09.884 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 13/36 2026-02-20T21:04:09.884 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/36 2026-02-20T21:04:09.884 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 15/36 2026-02-20T21:04:09.884 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 16/36 2026-02-20T21:04:09.884 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 17/36 2026-02-20T21:04:09.884 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 18/36 2026-02-20T21:04:09.884 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 19/36 2026-02-20T21:04:09.884 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 20/36 2026-02-20T21:04:09.884 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 21/36 2026-02-20T21:04:09.884 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 22/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 23/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 24/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 25/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 26/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 27/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 28/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 29/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 30/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : re2-1:20211101-20.el9.x86_64 31/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 32/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : librados2-2:20.2.0-677.gf7870454.el9.x86_64 33/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 34/36 2026-02-20T21:04:09.885 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : librbd1-2:20.2.0-677.gf7870454.el9.x86_64 35/36 2026-02-20T21:04:09.912 INFO:teuthology.orchestra.run.trial163.stdout:cephadm-20.2.0-677.gf7870454.el9.noarch.rpm 8.2 MB/s | 1.0 MB 00:00 2026-02-20T21:04:09.912 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:09.912 INFO:teuthology.orchestra.run.trial163.stdout:Total 8.1 MB/s | 1.0 MB 00:00 2026-02-20T21:04:09.912 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:09.915 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:09.915 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:09.919 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:09.919 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:09.937 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:09.944 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 36/36 2026-02-20T21:04:09.945 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:09.945 INFO:teuthology.orchestra.run.trial069.stdout:Upgraded: 2026-02-20T21:04:09.945 INFO:teuthology.orchestra.run.trial069.stdout: librados2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.945 INFO:teuthology.orchestra.run.trial069.stdout: librbd1-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.945 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:09.945 INFO:teuthology.orchestra.run.trial069.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-20T21:04:09.945 INFO:teuthology.orchestra.run.trial069.stdout: ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.945 INFO:teuthology.orchestra.run.trial069.stdout: ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.945 INFO:teuthology.orchestra.run.trial069.stdout: ceph-radosgw-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.945 INFO:teuthology.orchestra.run.trial069.stdout: ceph-selinux-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.946 INFO:teuthology.orchestra.run.trial069.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-20T21:04:09.946 INFO:teuthology.orchestra.run.trial069.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-20T21:04:09.946 INFO:teuthology.orchestra.run.trial069.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-20T21:04:09.946 INFO:teuthology.orchestra.run.trial069.stdout: libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.946 INFO:teuthology.orchestra.run.trial069.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-20T21:04:09.946 INFO:teuthology.orchestra.run.trial069.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-20T21:04:09.946 INFO:teuthology.orchestra.run.trial069.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-20T21:04:09.946 INFO:teuthology.orchestra.run.trial069.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-20T21:04:09.946 INFO:teuthology.orchestra.run.trial069.stdout: libradosstriper1-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.946 INFO:teuthology.orchestra.run.trial069.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-20T21:04:09.946 INFO:teuthology.orchestra.run.trial069.stdout: librgw2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.947 INFO:teuthology.orchestra.run.trial069.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-20T21:04:09.947 INFO:teuthology.orchestra.run.trial069.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-20T21:04:09.947 INFO:teuthology.orchestra.run.trial069.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-20T21:04:09.947 INFO:teuthology.orchestra.run.trial069.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-20T21:04:09.947 INFO:teuthology.orchestra.run.trial069.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-20T21:04:09.947 INFO:teuthology.orchestra.run.trial069.stdout: python3-ceph-argparse-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.947 INFO:teuthology.orchestra.run.trial069.stdout: python3-ceph-common-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.947 INFO:teuthology.orchestra.run.trial069.stdout: python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.947 INFO:teuthology.orchestra.run.trial069.stdout: python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.947 INFO:teuthology.orchestra.run.trial069.stdout: python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.947 INFO:teuthology.orchestra.run.trial069.stdout: python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:09.948 INFO:teuthology.orchestra.run.trial069.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-20T21:04:09.948 INFO:teuthology.orchestra.run.trial069.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-20T21:04:09.948 INFO:teuthology.orchestra.run.trial069.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-20T21:04:09.948 INFO:teuthology.orchestra.run.trial069.stdout: re2-1:20211101-20.el9.x86_64 2026-02-20T21:04:09.948 INFO:teuthology.orchestra.run.trial069.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-20T21:04:09.948 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:09.948 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:09.986 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install ceph-test 2026-02-20T21:04:09.999 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:04:10.012 INFO:teuthology.orchestra.run.trial163.stdout: Installing : cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:04:10.072 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:04:10.126 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:04:10.126 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:10.126 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:10.126 INFO:teuthology.orchestra.run.trial163.stdout: cephadm-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:10.126 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:10.126 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:10.153 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install ceph-immutable-object-cache 2026-02-20T21:04:10.290 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:00:32 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:10.372 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repository Size 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout: ceph-test x86_64 2:20.2.0-677.gf7870454.el9 ceph 84 M 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout:Installing dependencies: 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout: libcephsqlite x86_64 2:20.2.0-677.gf7870454.el9 ceph 164 k 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout:Install 5 Packages 2026-02-20T21:04:10.373 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:10.374 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 85 M 2026-02-20T21:04:10.374 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 365 M 2026-02-20T21:04:10.374 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:10.464 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:26 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:10.547 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:10.547 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:10.547 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repo Size 2026-02-20T21:04:10.547 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:10.547 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:10.547 INFO:teuthology.orchestra.run.trial163.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-677.gf7870454.el9 ceph 154 k 2026-02-20T21:04:10.548 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:10.548 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:10.548 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:10.548 INFO:teuthology.orchestra.run.trial163.stdout:Install 1 Package 2026-02-20T21:04:10.548 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:10.548 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 154 k 2026-02-20T21:04:10.548 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 447 k 2026-02-20T21:04:10.548 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:10.639 INFO:teuthology.orchestra.run.trial163.stdout:ceph-immutable-object-cache-20.2.0-677.gf787045 1.6 MB/s | 154 kB 00:00 2026-02-20T21:04:10.639 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:10.639 INFO:teuthology.orchestra.run.trial163.stdout:Total 1.6 MB/s | 154 kB 00:00 2026-02-20T21:04:10.639 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:10.644 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:10.644 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:10.670 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:10.670 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:10.719 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:10.720 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-immutable-object-cache-2:20.2.0-677.gf7870454.e 1/1 2026-02-20T21:04:10.743 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-677.gf7870454.e 1/1 2026-02-20T21:04:10.743 INFO:teuthology.orchestra.run.trial163.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:10.743 INFO:teuthology.orchestra.run.trial163.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-20T21:04:10.743 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:10.767 INFO:teuthology.orchestra.run.trial069.stdout:(1/5): libcephsqlite-20.2.0-677.gf7870454.el9.x 1.8 MB/s | 164 kB 00:00 2026-02-20T21:04:10.818 INFO:teuthology.orchestra.run.trial069.stdout:(2/5): libxslt-1.1.34-12.el9.x86_64.rpm 1.6 MB/s | 233 kB 00:00 2026-02-20T21:04:10.843 INFO:teuthology.orchestra.run.trial069.stdout:(3/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 2.5 MB/s | 64 kB 00:00 2026-02-20T21:04:10.881 INFO:teuthology.orchestra.run.trial069.stdout:(4/5): socat-1.7.4.1-8.el9.x86_64.rpm 2.6 MB/s | 303 kB 00:00 2026-02-20T21:04:11.030 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-677.gf7870454.e 1/1 2026-02-20T21:04:11.030 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:11.030 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:11.030 INFO:teuthology.orchestra.run.trial163.stdout: ceph-immutable-object-cache-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:11.030 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:11.030 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:11.058 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install ceph-mgr 2026-02-20T21:04:11.362 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:27 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:11.426 INFO:teuthology.orchestra.run.trial163.stdout:Package ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:11.444 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:11.445 INFO:teuthology.orchestra.run.trial163.stdout:Nothing to do. 2026-02-20T21:04:11.445 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:11.465 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install ceph-mgr-dashboard 2026-02-20T21:04:11.772 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:27 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:11.855 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:11.855 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:11.855 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repository Size 2026-02-20T21:04:11.855 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:11.855 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-dashboard noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 11 M 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout:Installing dependencies: 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: ceph-grafana-dashboards noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 43 k 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: ceph-prometheus-alerts noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 17 k 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: python3-jmespath noarch 1.0.1-1.el9 appstream 48 k 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout:Install 14 Packages 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 16 M 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 158 M 2026-02-20T21:04:11.856 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:12.493 INFO:teuthology.orchestra.run.trial163.stdout:(1/14): ceph-prometheus-alerts-20.2.0-677.gf787 151 kB/s | 17 kB 00:00 2026-02-20T21:04:12.504 INFO:teuthology.orchestra.run.trial163.stdout:(2/14): ceph-grafana-dashboards-20.2.0-677.gf78 343 kB/s | 43 kB 00:00 2026-02-20T21:04:12.577 INFO:teuthology.orchestra.run.trial069.stdout:(5/5): ceph-test-20.2.0-677.gf7870454.el9.x86_6 44 MB/s | 84 MB 00:01 2026-02-20T21:04:12.578 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:12.578 INFO:teuthology.orchestra.run.trial069.stdout:Total 38 MB/s | 85 MB 00:02 2026-02-20T21:04:12.602 INFO:teuthology.orchestra.run.trial163.stdout:(3/14): python3-jmespath-1.0.1-1.el9.noarch.rpm 485 kB/s | 48 kB 00:00 2026-02-20T21:04:12.604 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:12.614 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:12.614 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:12.691 INFO:teuthology.orchestra.run.trial163.stdout:(4/14): python3-protobuf-3.14.0-17.el9.noarch.r 3.0 MB/s | 267 kB 00:00 2026-02-20T21:04:12.705 INFO:teuthology.orchestra.run.trial163.stdout:(5/14): protobuf-3.14.0-17.el9.x86_64.rpm 4.8 MB/s | 1.0 MB 00:00 2026-02-20T21:04:12.823 INFO:teuthology.orchestra.run.trial163.stdout:(6/14): protobuf-compiler-3.14.0-17.el9.x86_64. 6.4 MB/s | 862 kB 00:00 2026-02-20T21:04:12.887 INFO:teuthology.orchestra.run.trial163.stdout:(7/14): abseil-cpp-20211102.0-4.el9.x86_64.rpm 3.0 MB/s | 551 kB 00:00 2026-02-20T21:04:12.899 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:12.900 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:12.913 INFO:teuthology.orchestra.run.trial163.stdout:(8/14): grpc-data-1.46.7-10.el9.noarch.rpm 216 kB/s | 19 kB 00:00 2026-02-20T21:04:12.970 INFO:teuthology.orchestra.run.trial163.stdout:(9/14): python3-grpcio-1.46.7-10.el9.x86_64.rpm 24 MB/s | 2.0 MB 00:00 2026-02-20T21:04:12.984 INFO:teuthology.orchestra.run.trial163.stdout:(10/14): ceph-mgr-dashboard-20.2.0-677.gf787045 17 MB/s | 11 MB 00:00 2026-02-20T21:04:12.995 INFO:teuthology.orchestra.run.trial163.stdout:(11/14): python3-repoze-lru-0.7-16.el9.noarch.r 1.2 MB/s | 31 kB 00:00 2026-02-20T21:04:13.020 INFO:teuthology.orchestra.run.trial163.stdout:(12/14): python3-xmltodict-0.12.0-15.el9.noarch 900 kB/s | 22 kB 00:00 2026-02-20T21:04:13.026 INFO:teuthology.orchestra.run.trial163.stdout:(13/14): python3-grpcio-tools-1.46.7-10.el9.x86 1.2 MB/s | 144 kB 00:00 2026-02-20T21:04:13.164 INFO:teuthology.orchestra.run.trial163.stdout:(14/14): python3-routes-2.5.1-5.el9.noarch.rpm 1.0 MB/s | 188 kB 00:00 2026-02-20T21:04:13.164 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:13.165 INFO:teuthology.orchestra.run.trial163.stdout:Total 12 MB/s | 16 MB 00:01 2026-02-20T21:04:13.200 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:13.210 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:13.210 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:13.211 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:13.217 INFO:teuthology.orchestra.run.trial069.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-20T21:04:13.221 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-20T21:04:13.229 INFO:teuthology.orchestra.run.trial069.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-20T21:04:13.230 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 4/5 2026-02-20T21:04:13.306 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:13.306 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:13.421 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:13.428 INFO:teuthology.orchestra.run.trial163.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 1/14 2026-02-20T21:04:13.432 INFO:teuthology.orchestra.run.trial163.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 2/14 2026-02-20T21:04:13.437 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 3/14 2026-02-20T21:04:13.448 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 4/14 2026-02-20T21:04:13.450 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 5/14 2026-02-20T21:04:13.477 INFO:teuthology.orchestra.run.trial163.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 6/14 2026-02-20T21:04:13.520 INFO:teuthology.orchestra.run.trial163.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 7/14 2026-02-20T21:04:13.549 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 8/14 2026-02-20T21:04:13.559 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 9/14 2026-02-20T21:04:13.566 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 10/14 2026-02-20T21:04:13.568 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-jmespath-1.0.1-1.el9.noarch 11/14 2026-02-20T21:04:13.575 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-prometheus-alerts-2:20.2.0-677.gf7870454.el9. 12/14 2026-02-20T21:04:14.708 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-grafana-dashboards-2:20.2.0-677.gf7870454.el9 13/14 2026-02-20T21:04:14.713 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noar 14/14 2026-02-20T21:04:14.906 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noar 14/14 2026-02-20T21:04:14.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-677.gf7870454.el9 1/14 2026-02-20T21:04:14.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noar 2/14 2026-02-20T21:04:14.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-677.gf7870454.el9. 3/14 2026-02-20T21:04:14.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 4/14 2026-02-20T21:04:14.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-jmespath-1.0.1-1.el9.noarch 5/14 2026-02-20T21:04:14.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 6/14 2026-02-20T21:04:14.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 7/14 2026-02-20T21:04:14.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 8/14 2026-02-20T21:04:14.906 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 9/14 2026-02-20T21:04:14.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 10/14 2026-02-20T21:04:14.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 11/14 2026-02-20T21:04:14.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 12/14 2026-02-20T21:04:14.907 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 13/14 2026-02-20T21:04:14.961 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 14/14 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: ceph-grafana-dashboards-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: ceph-prometheus-alerts-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-jmespath-1.0.1-1.el9.noarch 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:14.962 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:14.996 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-20T21:04:15.300 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:31 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:15.382 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout:====================================================================================== 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repository Size 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout:====================================================================================== 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 7.4 M 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout:Installing dependencies: 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:15.383 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:15.384 INFO:teuthology.orchestra.run.trial163.stdout:====================================================================================== 2026-02-20T21:04:15.384 INFO:teuthology.orchestra.run.trial163.stdout:Install 12 Packages 2026-02-20T21:04:15.384 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:15.384 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 43 M 2026-02-20T21:04:15.384 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 234 M 2026-02-20T21:04:15.384 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:15.819 INFO:teuthology.orchestra.run.trial163.stdout:(1/12): ceph-mgr-diskprediction-local-20.2.0-67 26 MB/s | 7.4 MB 00:00 2026-02-20T21:04:15.875 INFO:teuthology.orchestra.run.trial163.stdout:(2/12): flexiblas-3.0.4-9.el9.x86_64.rpm 526 kB/s | 30 kB 00:00 2026-02-20T21:04:16.010 INFO:teuthology.orchestra.run.trial163.stdout:(3/12): flexiblas-netlib-3.0.4-9.el9.x86_64.rpm 22 MB/s | 3.0 MB 00:00 2026-02-20T21:04:16.029 INFO:teuthology.orchestra.run.trial163.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-9.el9.x 759 kB/s | 15 kB 00:00 2026-02-20T21:04:16.050 INFO:teuthology.orchestra.run.trial163.stdout:(5/12): openblas-0.3.29-1.el9.x86_64.rpm 2.0 MB/s | 42 kB 00:00 2026-02-20T21:04:16.116 INFO:teuthology.orchestra.run.trial163.stdout:(6/12): openblas-openmp-0.3.29-1.el9.x86_64.rpm 80 MB/s | 5.3 MB 00:00 2026-02-20T21:04:16.139 INFO:teuthology.orchestra.run.trial163.stdout:(7/12): python3-devel-3.9.25-3.el9.x86_64.rpm 11 MB/s | 244 kB 00:00 2026-02-20T21:04:16.193 INFO:teuthology.orchestra.run.trial163.stdout:(8/12): python3-numpy-1.23.5-2.el9.x86_64.rpm 114 MB/s | 6.1 MB 00:00 2026-02-20T21:04:16.216 INFO:teuthology.orchestra.run.trial163.stdout:(9/12): python3-numpy-f2py-1.23.5-2.el9.x86_64. 19 MB/s | 442 kB 00:00 2026-02-20T21:04:16.261 INFO:teuthology.orchestra.run.trial163.stdout:(10/12): libquadmath-11.5.0-14.el9.x86_64.rpm 254 kB/s | 184 kB 00:00 2026-02-20T21:04:16.367 INFO:teuthology.orchestra.run.trial163.stdout:(11/12): python3-scipy-1.9.3-2.el9.x86_64.rpm 128 MB/s | 19 MB 00:00 2026-02-20T21:04:16.391 INFO:teuthology.orchestra.run.trial163.stdout:(12/12): libgfortran-11.5.0-14.el9.x86_64.rpm 927 kB/s | 794 kB 00:00 2026-02-20T21:04:16.392 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:16.392 INFO:teuthology.orchestra.run.trial163.stdout:Total 42 MB/s | 43 MB 00:01 2026-02-20T21:04:16.549 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:16.562 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:16.562 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:16.761 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:16.761 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:16.942 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 4/5 2026-02-20T21:04:16.960 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 5/5 2026-02-20T21:04:16.976 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:16.979 INFO:teuthology.orchestra.run.trial163.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 1/12 2026-02-20T21:04:16.985 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 2/12 2026-02-20T21:04:17.018 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 3/12 2026-02-20T21:04:17.023 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 4/12 2026-02-20T21:04:17.082 INFO:teuthology.orchestra.run.trial163.stdout: Installing : openblas-0.3.29-1.el9.x86_64 5/12 2026-02-20T21:04:17.085 INFO:teuthology.orchestra.run.trial163.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 6/12 2026-02-20T21:04:17.106 INFO:teuthology.orchestra.run.trial163.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 7/12 2026-02-20T21:04:17.291 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 5/5 2026-02-20T21:04:17.292 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 1/5 2026-02-20T21:04:17.292 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 2/5 2026-02-20T21:04:17.292 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-20T21:04:17.292 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-20T21:04:17.345 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-20T21:04:17.345 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:17.346 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:17.346 INFO:teuthology.orchestra.run.trial069.stdout: ceph-test-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:17.346 INFO:teuthology.orchestra.run.trial069.stdout: libcephsqlite-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:17.346 INFO:teuthology.orchestra.run.trial069.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-20T21:04:17.346 INFO:teuthology.orchestra.run.trial069.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-20T21:04:17.346 INFO:teuthology.orchestra.run.trial069.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-20T21:04:17.347 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:17.347 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:17.380 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install ceph 2026-02-20T21:04:17.429 INFO:teuthology.orchestra.run.trial163.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 8/12 2026-02-20T21:04:17.502 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 9/12 2026-02-20T21:04:17.684 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:00:39 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:17.769 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:17.770 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:17.771 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repository Size 2026-02-20T21:04:17.771 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:17.771 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:17.771 INFO:teuthology.orchestra.run.trial069.stdout: ceph x86_64 2:20.2.0-677.gf7870454.el9 ceph 6.5 k 2026-02-20T21:04:17.771 INFO:teuthology.orchestra.run.trial069.stdout:Installing dependencies: 2026-02-20T21:04:17.771 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mds x86_64 2:20.2.0-677.gf7870454.el9 ceph 2.3 M 2026-02-20T21:04:17.771 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mgr x86_64 2:20.2.0-677.gf7870454.el9 ceph 961 k 2026-02-20T21:04:17.771 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mgr-modules-core noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 290 k 2026-02-20T21:04:17.771 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mon x86_64 2:20.2.0-677.gf7870454.el9 ceph 5.0 M 2026-02-20T21:04:17.772 INFO:teuthology.orchestra.run.trial069.stdout: ceph-osd x86_64 2:20.2.0-677.gf7870454.el9 ceph 17 M 2026-02-20T21:04:17.772 INFO:teuthology.orchestra.run.trial069.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-20T21:04:17.772 INFO:teuthology.orchestra.run.trial069.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-20T21:04:17.772 INFO:teuthology.orchestra.run.trial069.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-20T21:04:17.772 INFO:teuthology.orchestra.run.trial069.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-20T21:04:17.772 INFO:teuthology.orchestra.run.trial069.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-20T21:04:17.772 INFO:teuthology.orchestra.run.trial069.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-20T21:04:17.772 INFO:teuthology.orchestra.run.trial069.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-20T21:04:17.772 INFO:teuthology.orchestra.run.trial069.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-20T21:04:17.772 INFO:teuthology.orchestra.run.trial069.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-20T21:04:17.772 INFO:teuthology.orchestra.run.trial069.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-20T21:04:17.772 INFO:teuthology.orchestra.run.trial069.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-20T21:04:17.773 INFO:teuthology.orchestra.run.trial069.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-20T21:04:17.774 INFO:teuthology.orchestra.run.trial069.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-20T21:04:17.774 INFO:teuthology.orchestra.run.trial069.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-20T21:04:17.774 INFO:teuthology.orchestra.run.trial069.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-20T21:04:17.774 INFO:teuthology.orchestra.run.trial069.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-20T21:04:17.774 INFO:teuthology.orchestra.run.trial069.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-20T21:04:17.774 INFO:teuthology.orchestra.run.trial069.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-20T21:04:17.774 INFO:teuthology.orchestra.run.trial069.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-20T21:04:17.774 INFO:teuthology.orchestra.run.trial069.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-20T21:04:17.774 INFO:teuthology.orchestra.run.trial069.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-20T21:04:17.774 INFO:teuthology.orchestra.run.trial069.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-20T21:04:17.774 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:17.774 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:17.775 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:17.775 INFO:teuthology.orchestra.run.trial069.stdout:Install 40 Packages 2026-02-20T21:04:17.775 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:17.775 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 30 M 2026-02-20T21:04:17.776 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 100 M 2026-02-20T21:04:17.776 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:18.172 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 10/12 2026-02-20T21:04:18.418 INFO:teuthology.orchestra.run.trial069.stdout:(1/40): ceph-20.2.0-677.gf7870454.el9.x86_64.rp 114 kB/s | 6.5 kB 00:00 2026-02-20T21:04:18.489 INFO:teuthology.orchestra.run.trial069.stdout:(2/40): ceph-mgr-20.2.0-677.gf7870454.el9.x86_6 7.4 MB/s | 961 kB 00:00 2026-02-20T21:04:18.531 INFO:teuthology.orchestra.run.trial069.stdout:(3/40): ceph-mds-20.2.0-677.gf7870454.el9.x86_6 14 MB/s | 2.3 MB 00:00 2026-02-20T21:04:18.555 INFO:teuthology.orchestra.run.trial069.stdout:(4/40): ceph-mgr-modules-core-20.2.0-677.gf7870 12 MB/s | 290 kB 00:00 2026-02-20T21:04:18.631 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 11/12 2026-02-20T21:04:18.632 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-mgr-diskprediction-local-2:20.2.0-677.gf78704 12/12 2026-02-20T21:04:18.666 INFO:teuthology.orchestra.run.trial069.stdout:(5/40): ceph-mon-20.2.0-677.gf7870454.el9.x86_6 20 MB/s | 5.0 MB 00:00 2026-02-20T21:04:18.901 INFO:teuthology.orchestra.run.trial069.stdout:(6/40): ceph-osd-20.2.0-677.gf7870454.el9.x86_6 41 MB/s | 17 MB 00:00 2026-02-20T21:04:18.937 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-677.gf78704 12/12 2026-02-20T21:04:18.938 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-677.gf78704 1/12 2026-02-20T21:04:18.938 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 2/12 2026-02-20T21:04:18.938 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 3/12 2026-02-20T21:04:18.938 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 4/12 2026-02-20T21:04:18.938 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 5/12 2026-02-20T21:04:18.938 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 6/12 2026-02-20T21:04:18.938 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 7/12 2026-02-20T21:04:18.938 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 8/12 2026-02-20T21:04:18.938 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 9/12 2026-02-20T21:04:18.938 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 10/12 2026-02-20T21:04:18.938 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 11/12 2026-02-20T21:04:18.996 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 12/12 2026-02-20T21:04:18.996 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:18.996 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:18.996 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-diskprediction-local-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:18.996 INFO:teuthology.orchestra.run.trial163.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-20T21:04:18.996 INFO:teuthology.orchestra.run.trial163.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-20T21:04:18.996 INFO:teuthology.orchestra.run.trial163.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-20T21:04:18.997 INFO:teuthology.orchestra.run.trial163.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-20T21:04:18.997 INFO:teuthology.orchestra.run.trial163.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-20T21:04:18.997 INFO:teuthology.orchestra.run.trial163.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-20T21:04:18.997 INFO:teuthology.orchestra.run.trial163.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-20T21:04:18.997 INFO:teuthology.orchestra.run.trial163.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-20T21:04:18.997 INFO:teuthology.orchestra.run.trial163.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-20T21:04:18.997 INFO:teuthology.orchestra.run.trial163.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-20T21:04:18.997 INFO:teuthology.orchestra.run.trial163.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-20T21:04:18.997 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:18.997 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:19.010 INFO:teuthology.orchestra.run.trial069.stdout:(7/40): ledmon-libs-1.1.0-3.el9.x86_64.rpm 89 kB/s | 40 kB 00:00 2026-02-20T21:04:19.033 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install ceph-mgr-rook 2026-02-20T21:04:19.151 INFO:teuthology.orchestra.run.trial069.stdout:(8/40): libconfig-1.7.2-9.el9.x86_64.rpm 148 kB/s | 72 kB 00:00 2026-02-20T21:04:19.315 INFO:teuthology.orchestra.run.trial069.stdout:(9/40): python3-ply-3.11-14.el9.noarch.rpm 649 kB/s | 106 kB 00:00 2026-02-20T21:04:19.341 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:35 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:19.426 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:19.426 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:19.426 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repository Size 2026-02-20T21:04:19.426 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-rook noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 50 k 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout:Installing dependencies: 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout:Install 10 Packages 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 2.0 M 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 25 M 2026-02-20T21:04:19.427 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:19.558 INFO:teuthology.orchestra.run.trial069.stdout:(10/40): python3-pycparser-2.20-6.el9.noarch.rp 555 kB/s | 135 kB 00:00 2026-02-20T21:04:19.704 INFO:teuthology.orchestra.run.trial163.stdout:(1/10): ceph-mgr-rook-20.2.0-677.gf7870454.el9. 670 kB/s | 50 kB 00:00 2026-02-20T21:04:19.801 INFO:teuthology.orchestra.run.trial069.stdout:(11/40): python3-pyparsing-2.4.7-9.el9.noarch.r 620 kB/s | 150 kB 00:00 2026-02-20T21:04:19.966 INFO:teuthology.orchestra.run.trial069.stdout:(12/40): python3-requests-2.25.1-10.el9.noarch. 770 kB/s | 126 kB 00:00 2026-02-20T21:04:20.079 INFO:teuthology.orchestra.run.trial069.stdout:(13/40): python3-cffi-1.14.5-5.el9.x86_64.rpm 215 kB/s | 253 kB 00:01 2026-02-20T21:04:20.288 INFO:teuthology.orchestra.run.trial069.stdout:(14/40): python3-urllib3-1.26.5-7.el9.noarch.rp 676 kB/s | 218 kB 00:00 2026-02-20T21:04:20.341 INFO:teuthology.orchestra.run.trial163.stdout:(2/10): python3-requests-oauthlib-1.3.0-12.el9. 84 kB/s | 54 kB 00:00 2026-02-20T21:04:20.414 INFO:teuthology.orchestra.run.trial163.stdout:(3/10): python3-pyasn1-0.4.8-6.el9.noarch.rpm 202 kB/s | 159 kB 00:00 2026-02-20T21:04:20.452 INFO:teuthology.orchestra.run.trial163.stdout:(4/10): python3-cachetools-4.2.4-1.el9.noarch.r 290 kB/s | 32 kB 00:00 2026-02-20T21:04:20.496 INFO:teuthology.orchestra.run.trial163.stdout:(5/10): python3-certifi-2023.05.07-4.el9.noarch 173 kB/s | 14 kB 00:00 2026-02-20T21:04:20.500 INFO:teuthology.orchestra.run.trial163.stdout:(6/10): python3-pyasn1-modules-0.4.8-6.el9.noar 320 kB/s | 279 kB 00:00 2026-02-20T21:04:20.521 INFO:teuthology.orchestra.run.trial069.stdout:(15/40): python3-cryptography-36.0.1-5.el9.x86_ 844 kB/s | 1.2 MB 00:01 2026-02-20T21:04:20.550 INFO:teuthology.orchestra.run.trial069.stdout:(16/40): unzip-6.0-59.el9.x86_64.rpm 386 kB/s | 182 kB 00:00 2026-02-20T21:04:20.602 INFO:teuthology.orchestra.run.trial163.stdout:(7/10): python3-google-auth-2.45.0-1.el9.noarch 1.7 MB/s | 254 kB 00:00 2026-02-20T21:04:20.610 INFO:teuthology.orchestra.run.trial069.stdout:(17/40): zip-3.0-35.el9.x86_64.rpm 826 kB/s | 266 kB 00:00 2026-02-20T21:04:20.610 INFO:teuthology.orchestra.run.trial069.stdout:(18/40): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 2.7 MB/s | 246 kB 00:00 2026-02-20T21:04:20.620 INFO:teuthology.orchestra.run.trial069.stdout:(19/40): lua-5.4.4-4.el9.x86_64.rpm 2.7 MB/s | 188 kB 00:00 2026-02-20T21:04:20.640 INFO:teuthology.orchestra.run.trial069.stdout:(20/40): python3-libstoragemgmt-1.10.1-1.el9.x8 5.7 MB/s | 177 kB 00:00 2026-02-20T21:04:20.641 INFO:teuthology.orchestra.run.trial069.stdout:(21/40): python3-packaging-20.9-5.el9.noarch.rp 2.5 MB/s | 77 kB 00:00 2026-02-20T21:04:20.650 INFO:teuthology.orchestra.run.trial069.stdout:(22/40): python3-toml-0.10.2-6.el9.noarch.rpm 1.3 MB/s | 42 kB 00:00 2026-02-20T21:04:20.717 INFO:teuthology.orchestra.run.trial163.stdout:(8/10): python3-kubernetes-26.1.0-3.el9.noarch. 4.6 MB/s | 1.0 MB 00:00 2026-02-20T21:04:20.725 INFO:teuthology.orchestra.run.trial069.stdout:(23/40): lua-devel-5.4.4-4.el9.x86_64.rpm 263 kB/s | 22 kB 00:00 2026-02-20T21:04:20.757 INFO:teuthology.orchestra.run.trial163.stdout:(9/10): python3-rsa-4.9-2.el9.noarch.rpm 230 kB/s | 59 kB 00:00 2026-02-20T21:04:20.865 INFO:teuthology.orchestra.run.trial069.stdout:(24/40): python3-autocommand-2.2.2-8.el9.noarch 137 kB/s | 29 kB 00:00 2026-02-20T21:04:20.996 INFO:teuthology.orchestra.run.trial069.stdout:(25/40): luarocks-3.9.2-5.el9.noarch.rpm 427 kB/s | 151 kB 00:00 2026-02-20T21:04:21.031 INFO:teuthology.orchestra.run.trial069.stdout:(26/40): python3-backports-tarfile-1.2.0-1.el9. 196 kB/s | 60 kB 00:00 2026-02-20T21:04:21.035 INFO:teuthology.orchestra.run.trial163.stdout:(10/10): python3-websocket-client-1.2.3-2.el9.n 207 kB/s | 90 kB 00:00 2026-02-20T21:04:21.037 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:21.037 INFO:teuthology.orchestra.run.trial163.stdout:Total 1.2 MB/s | 2.0 MB 00:01 2026-02-20T21:04:21.061 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:21.070 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:21.070 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:21.074 INFO:teuthology.orchestra.run.trial069.stdout:(27/40): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 208 kB/s | 43 kB 00:00 2026-02-20T21:04:21.108 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:21.108 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:21.181 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:21.215 INFO:teuthology.orchestra.run.trial069.stdout:(28/40): python3-jaraco-8.2.1-3.el9.noarch.rpm 76 kB/s | 11 kB 00:00 2026-02-20T21:04:21.232 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/10 2026-02-20T21:04:21.244 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/10 2026-02-20T21:04:21.259 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/10 2026-02-20T21:04:21.263 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/10 2026-02-20T21:04:21.271 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/10 2026-02-20T21:04:21.310 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/10 2026-02-20T21:04:21.322 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 7/10 2026-02-20T21:04:21.358 INFO:teuthology.orchestra.run.trial069.stdout:(29/40): python3-cheroot-10.0.1-4.el9.noarch.rp 478 kB/s | 173 kB 00:00 2026-02-20T21:04:21.430 INFO:teuthology.orchestra.run.trial069.stdout:(30/40): python3-jaraco-classes-3.2.1-5.el9.noa 82 kB/s | 18 kB 00:00 2026-02-20T21:04:21.466 INFO:teuthology.orchestra.run.trial069.stdout:(31/40): python3-cherrypy-18.6.1-2.el9.noarch.r 825 kB/s | 358 kB 00:00 2026-02-20T21:04:21.568 INFO:teuthology.orchestra.run.trial069.stdout:(32/40): python3-jaraco-collections-3.0.0-8.el9 110 kB/s | 23 kB 00:00 2026-02-20T21:04:21.638 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 8/10 2026-02-20T21:04:21.639 INFO:teuthology.orchestra.run.trial069.stdout:(33/40): python3-jaraco-context-6.0.1-3.el9.noa 94 kB/s | 20 kB 00:00 2026-02-20T21:04:21.653 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 9/10 2026-02-20T21:04:21.654 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 10/10 2026-02-20T21:04:21.683 INFO:teuthology.orchestra.run.trial069.stdout:(34/40): python3-jaraco-functools-3.5.0-2.el9.n 89 kB/s | 19 kB 00:00 2026-02-20T21:04:21.762 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 10/10 2026-02-20T21:04:21.762 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 1/10 2026-02-20T21:04:21.762 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 2/10 2026-02-20T21:04:21.762 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 3/10 2026-02-20T21:04:21.763 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 4/10 2026-02-20T21:04:21.763 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 5/10 2026-02-20T21:04:21.763 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 6/10 2026-02-20T21:04:21.763 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 7/10 2026-02-20T21:04:21.763 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 8/10 2026-02-20T21:04:21.763 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 9/10 2026-02-20T21:04:21.784 INFO:teuthology.orchestra.run.trial069.stdout:(35/40): python3-jaraco-text-4.0.0-2.el9.noarch 123 kB/s | 26 kB 00:00 2026-02-20T21:04:21.817 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 10/10 2026-02-20T21:04:21.817 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:21.817 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:21.817 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:21.818 INFO:teuthology.orchestra.run.trial163.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-20T21:04:21.818 INFO:teuthology.orchestra.run.trial163.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-20T21:04:21.818 INFO:teuthology.orchestra.run.trial163.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-20T21:04:21.818 INFO:teuthology.orchestra.run.trial163.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-20T21:04:21.818 INFO:teuthology.orchestra.run.trial163.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-20T21:04:21.818 INFO:teuthology.orchestra.run.trial163.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-20T21:04:21.818 INFO:teuthology.orchestra.run.trial163.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-20T21:04:21.818 INFO:teuthology.orchestra.run.trial163.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-20T21:04:21.818 INFO:teuthology.orchestra.run.trial163.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-20T21:04:21.818 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:21.818 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:21.848 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install ceph-mgr-cephadm 2026-02-20T21:04:21.898 INFO:teuthology.orchestra.run.trial069.stdout:(36/40): python3-portend-3.1.0-2.el9.noarch.rpm 76 kB/s | 16 kB 00:00 2026-02-20T21:04:21.929 INFO:teuthology.orchestra.run.trial069.stdout:(37/40): python3-more-itertools-8.12.0-2.el9.no 272 kB/s | 79 kB 00:00 2026-02-20T21:04:22.106 INFO:teuthology.orchestra.run.trial069.stdout:(38/40): python3-tempora-5.0.0-2.el9.noarch.rpm 172 kB/s | 36 kB 00:00 2026-02-20T21:04:22.114 INFO:teuthology.orchestra.run.trial069.stdout:(39/40): python3-pyOpenSSL-21.0.0-1.el9.noarch. 273 kB/s | 90 kB 00:00 2026-02-20T21:04:22.139 INFO:teuthology.orchestra.run.trial069.stdout:(40/40): python3-zc-lockfile-2.0-10.el9.noarch. 95 kB/s | 20 kB 00:00 2026-02-20T21:04:22.147 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:22.147 INFO:teuthology.orchestra.run.trial069.stdout:Total 6.9 MB/s | 30 MB 00:04 2026-02-20T21:04:22.153 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:38 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:22.236 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:22.237 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:22.237 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repository Size 2026-02-20T21:04:22.238 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:22.238 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:22.238 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-cephadm noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 173 k 2026-02-20T21:04:22.238 INFO:teuthology.orchestra.run.trial163.stdout:Installing dependencies: 2026-02-20T21:04:22.238 INFO:teuthology.orchestra.run.trial163.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-20T21:04:22.238 INFO:teuthology.orchestra.run.trial163.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-20T21:04:22.238 INFO:teuthology.orchestra.run.trial163.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-20T21:04:22.238 INFO:teuthology.orchestra.run.trial163.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-20T21:04:22.238 INFO:teuthology.orchestra.run.trial163.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-20T21:04:22.238 INFO:teuthology.orchestra.run.trial163.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-20T21:04:22.239 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:22.239 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:22.239 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:22.239 INFO:teuthology.orchestra.run.trial163.stdout:Install 7 Packages 2026-02-20T21:04:22.239 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:22.239 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 7.1 M 2026-02-20T21:04:22.239 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 33 M 2026-02-20T21:04:22.239 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:22.282 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:22.300 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:22.301 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:22.444 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:22.444 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:22.634 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:22.636 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/40 2026-02-20T21:04:22.640 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/40 2026-02-20T21:04:22.645 INFO:teuthology.orchestra.run.trial069.stdout: Installing : lua-5.4.4-4.el9.x86_64 3/40 2026-02-20T21:04:22.647 INFO:teuthology.orchestra.run.trial069.stdout: Installing : unzip-6.0-59.el9.x86_64 4/40 2026-02-20T21:04:22.653 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 5/40 2026-02-20T21:04:22.677 INFO:teuthology.orchestra.run.trial069.stdout: Installing : zip-3.0-35.el9.x86_64 6/40 2026-02-20T21:04:22.681 INFO:teuthology.orchestra.run.trial069.stdout: Installing : luarocks-3.9.2-5.el9.noarch 7/40 2026-02-20T21:04:22.685 INFO:teuthology.orchestra.run.trial069.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 8/40 2026-02-20T21:04:22.690 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 9/40 2026-02-20T21:04:22.696 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 10/40 2026-02-20T21:04:22.700 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 11/40 2026-02-20T21:04:22.706 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 12/40 2026-02-20T21:04:22.711 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 13/40 2026-02-20T21:04:22.714 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 14/40 2026-02-20T21:04:22.720 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 15/40 2026-02-20T21:04:22.724 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 16/40 2026-02-20T21:04:22.727 INFO:teuthology.orchestra.run.trial163.stdout:(1/7): ceph-mgr-cephadm-20.2.0-677.gf7870454.el 2.0 MB/s | 173 kB 00:00 2026-02-20T21:04:22.730 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 17/40 2026-02-20T21:04:22.734 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 18/40 2026-02-20T21:04:22.756 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 19/40 2026-02-20T21:04:22.769 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 20/40 2026-02-20T21:04:22.774 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 21/40 2026-02-20T21:04:22.783 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 22/40 2026-02-20T21:04:22.790 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-packaging-20.9-5.el9.noarch 23/40 2026-02-20T21:04:22.803 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-ply-3.11-14.el9.noarch 24/40 2026-02-20T21:04:22.818 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 25/40 2026-02-20T21:04:22.894 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 26/40 2026-02-20T21:04:22.902 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 27/40 2026-02-20T21:04:22.925 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 28/40 2026-02-20T21:04:22.954 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 29/40 2026-02-20T21:04:22.960 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 30/40 2026-02-20T21:04:23.006 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 31/40 2026-02-20T21:04:23.049 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-mgr-modules-core-2:20.2.0-677.gf7870454.el9.n 32/40 2026-02-20T21:04:23.050 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 33/40 2026-02-20T21:04:23.073 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 33/40 2026-02-20T21:04:23.073 INFO:teuthology.orchestra.run.trial069.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:23.074 INFO:teuthology.orchestra.run.trial069.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-20T21:04:23.074 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-20T21:04:23.074 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-20T21:04:23.074 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:23.078 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 34/40 2026-02-20T21:04:23.094 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 35/40 2026-02-20T21:04:23.094 INFO:teuthology.orchestra.run.trial069.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-20T21:04:23.094 INFO:teuthology.orchestra.run.trial069.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-20T21:04:23.094 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:23.104 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 35/40 2026-02-20T21:04:23.127 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 35/40 2026-02-20T21:04:23.127 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-20T21:04:23.127 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:23.283 INFO:teuthology.orchestra.run.trial163.stdout:(2/7): python3-markupsafe-1.1.1-12.el9.x86_64.r 62 kB/s | 35 kB 00:00 2026-02-20T21:04:23.498 INFO:teuthology.orchestra.run.trial163.stdout:(3/7): python3-asyncssh-2.13.2-5.el9.noarch.rpm 2.5 MB/s | 548 kB 00:00 2026-02-20T21:04:23.518 INFO:teuthology.orchestra.run.trial163.stdout:(4/7): python3-jinja2-2.11.3-8.el9.noarch.rpm 284 kB/s | 249 kB 00:00 2026-02-20T21:04:23.536 INFO:teuthology.orchestra.run.trial163.stdout:(5/7): python3-natsort-7.1.1-5.el9.noarch.rpm 1.5 MB/s | 58 kB 00:00 2026-02-20T21:04:23.639 INFO:teuthology.orchestra.run.trial163.stdout:(6/7): python3-typing-extensions-4.15.0-1.el9.n 716 kB/s | 86 kB 00:00 2026-02-20T21:04:23.812 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 36/40 2026-02-20T21:04:23.813 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 37/40 2026-02-20T21:04:23.839 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 37/40 2026-02-20T21:04:23.839 INFO:teuthology.orchestra.run.trial069.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:23.839 INFO:teuthology.orchestra.run.trial069.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-20T21:04:23.839 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-20T21:04:23.839 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-20T21:04:23.840 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:24.054 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 38/40 2026-02-20T21:04:24.075 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 38/40 2026-02-20T21:04:24.075 INFO:teuthology.orchestra.run.trial069.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:24.075 INFO:teuthology.orchestra.run.trial069.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-20T21:04:24.075 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-20T21:04:24.075 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-20T21:04:24.076 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:24.176 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 39/40 2026-02-20T21:04:24.197 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 39/40 2026-02-20T21:04:24.197 INFO:teuthology.orchestra.run.trial069.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:24.198 INFO:teuthology.orchestra.run.trial069.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-20T21:04:24.198 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-20T21:04:24.198 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-20T21:04:24.198 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:24.215 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-2:20.2.0-677.gf7870454.el9.x86_64 40/40 2026-02-20T21:04:24.544 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-2:20.2.0-677.gf7870454.el9.x86_64 40/40 2026-02-20T21:04:24.544 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-2:20.2.0-677.gf7870454.el9.x86_64 1/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 2/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 3/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 4/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 5/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-mgr-modules-core-2:20.2.0-677.gf7870454.el9.n 6/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 7/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 8/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 9/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 10/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-ply-3.11-14.el9.noarch 11/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 12/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 13/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 14/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 15/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : unzip-6.0-59.el9.x86_64 16/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : zip-3.0-35.el9.x86_64 17/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 18/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : lua-5.4.4-4.el9.x86_64 19/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 20/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 21/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 22/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 23/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 24/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 25/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 26/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 27/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 28/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 29/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 30/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 31/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 32/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 33/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 34/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 35/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 36/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 37/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 38/40 2026-02-20T21:04:24.545 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 39/40 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 40/40 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout: ceph-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mds-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mgr-modules-core-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mon-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout: ceph-osd-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-20T21:04:24.602 INFO:teuthology.orchestra.run.trial069.stdout: lua-5.4.4-4.el9.x86_64 2026-02-20T21:04:24.603 INFO:teuthology.orchestra.run.trial069.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-20T21:04:24.603 INFO:teuthology.orchestra.run.trial069.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-20T21:04:24.603 INFO:teuthology.orchestra.run.trial069.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-20T21:04:24.603 INFO:teuthology.orchestra.run.trial069.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-20T21:04:24.603 INFO:teuthology.orchestra.run.trial069.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-20T21:04:24.603 INFO:teuthology.orchestra.run.trial069.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-20T21:04:24.603 INFO:teuthology.orchestra.run.trial069.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-20T21:04:24.603 INFO:teuthology.orchestra.run.trial069.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-20T21:04:24.603 INFO:teuthology.orchestra.run.trial069.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-20T21:04:24.603 INFO:teuthology.orchestra.run.trial069.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-20T21:04:24.603 INFO:teuthology.orchestra.run.trial069.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-20T21:04:24.603 INFO:teuthology.orchestra.run.trial069.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-20T21:04:24.604 INFO:teuthology.orchestra.run.trial069.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-20T21:04:24.604 INFO:teuthology.orchestra.run.trial069.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-20T21:04:24.604 INFO:teuthology.orchestra.run.trial069.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-20T21:04:24.604 INFO:teuthology.orchestra.run.trial069.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-20T21:04:24.604 INFO:teuthology.orchestra.run.trial069.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-20T21:04:24.604 INFO:teuthology.orchestra.run.trial069.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-20T21:04:24.604 INFO:teuthology.orchestra.run.trial069.stdout: python3-ply-3.11-14.el9.noarch 2026-02-20T21:04:24.604 INFO:teuthology.orchestra.run.trial069.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-20T21:04:24.604 INFO:teuthology.orchestra.run.trial069.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-20T21:04:24.604 INFO:teuthology.orchestra.run.trial069.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-20T21:04:24.604 INFO:teuthology.orchestra.run.trial069.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-20T21:04:24.604 INFO:teuthology.orchestra.run.trial069.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-20T21:04:24.605 INFO:teuthology.orchestra.run.trial069.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-20T21:04:24.605 INFO:teuthology.orchestra.run.trial069.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-20T21:04:24.605 INFO:teuthology.orchestra.run.trial069.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-20T21:04:24.605 INFO:teuthology.orchestra.run.trial069.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-20T21:04:24.605 INFO:teuthology.orchestra.run.trial069.stdout: unzip-6.0-59.el9.x86_64 2026-02-20T21:04:24.605 INFO:teuthology.orchestra.run.trial069.stdout: zip-3.0-35.el9.x86_64 2026-02-20T21:04:24.605 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:24.605 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:24.636 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install ceph-base 2026-02-20T21:04:24.940 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:00:46 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:25.005 INFO:teuthology.orchestra.run.trial069.stdout:Package ceph-base-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:25.022 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:25.023 INFO:teuthology.orchestra.run.trial069.stdout:Nothing to do. 2026-02-20T21:04:25.023 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:25.043 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install cephadm 2026-02-20T21:04:25.344 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:00:47 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repository Size 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout: cephadm noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 1.0 M 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout:Install 1 Package 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 1.0 M 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 1.0 M 2026-02-20T21:04:25.426 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:25.551 INFO:teuthology.orchestra.run.trial069.stdout:cephadm-20.2.0-677.gf7870454.el9.noarch.rpm 8.1 MB/s | 1.0 MB 00:00 2026-02-20T21:04:25.551 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:25.552 INFO:teuthology.orchestra.run.trial069.stdout:Total 8.0 MB/s | 1.0 MB 00:00 2026-02-20T21:04:25.552 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:25.554 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:25.554 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:25.558 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:25.558 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:25.576 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:25.637 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:04:25.650 INFO:teuthology.orchestra.run.trial069.stdout: Installing : cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:04:25.711 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:04:25.765 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/1 2026-02-20T21:04:25.766 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:25.766 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:25.766 INFO:teuthology.orchestra.run.trial069.stdout: cephadm-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:25.766 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:25.766 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:25.793 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install ceph-immutable-object-cache 2026-02-20T21:04:26.107 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:00:48 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:26.191 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:26.192 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:26.192 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repo Size 2026-02-20T21:04:26.192 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:26.192 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:26.192 INFO:teuthology.orchestra.run.trial069.stdout: ceph-immutable-object-cache x86_64 2:20.2.0-677.gf7870454.el9 ceph 154 k 2026-02-20T21:04:26.192 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:26.192 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:26.192 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:26.192 INFO:teuthology.orchestra.run.trial069.stdout:Install 1 Package 2026-02-20T21:04:26.192 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:26.193 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 154 k 2026-02-20T21:04:26.193 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 447 k 2026-02-20T21:04:26.193 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:26.279 INFO:teuthology.orchestra.run.trial069.stdout:ceph-immutable-object-cache-20.2.0-677.gf787045 1.7 MB/s | 154 kB 00:00 2026-02-20T21:04:26.280 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:26.280 INFO:teuthology.orchestra.run.trial069.stdout:Total 1.7 MB/s | 154 kB 00:00 2026-02-20T21:04:26.280 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:26.284 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:26.284 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:26.310 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:26.310 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:26.361 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:26.362 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-immutable-object-cache-2:20.2.0-677.gf7870454.e 1/1 2026-02-20T21:04:26.386 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-immutable-object-cache-2:20.2.0-677.gf7870454.e 1/1 2026-02-20T21:04:26.386 INFO:teuthology.orchestra.run.trial069.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:26.386 INFO:teuthology.orchestra.run.trial069.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-20T21:04:26.386 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:26.638 INFO:teuthology.orchestra.run.trial163.stdout:(7/7): python3-babel-2.9.1-2.el9.noarch.rpm 1.5 MB/s | 6.0 MB 00:03 2026-02-20T21:04:26.640 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:26.640 INFO:teuthology.orchestra.run.trial163.stdout:Total 1.6 MB/s | 7.1 MB 00:04 2026-02-20T21:04:26.677 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:26.680 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-immutable-object-cache-2:20.2.0-677.gf7870454.e 1/1 2026-02-20T21:04:26.680 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:26.680 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:26.680 INFO:teuthology.orchestra.run.trial069.stdout: ceph-immutable-object-cache-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:26.681 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:26.681 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:26.685 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:26.685 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:26.708 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install ceph-mgr 2026-02-20T21:04:26.755 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:26.756 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:26.845 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:26.894 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 1/7 2026-02-20T21:04:26.904 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/7 2026-02-20T21:04:26.909 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/7 2026-02-20T21:04:27.008 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:00:49 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:27.072 INFO:teuthology.orchestra.run.trial069.stdout:Package ceph-mgr-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:27.089 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:27.090 INFO:teuthology.orchestra.run.trial069.stdout:Nothing to do. 2026-02-20T21:04:27.090 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:27.109 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install ceph-mgr-dashboard 2026-02-20T21:04:27.131 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 4/7 2026-02-20T21:04:27.155 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 5/7 2026-02-20T21:04:27.178 INFO:teuthology.orchestra.run.trial163.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 6/7 2026-02-20T21:04:27.180 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 7/7 2026-02-20T21:04:27.287 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 7/7 2026-02-20T21:04:27.287 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/7 2026-02-20T21:04:27.287 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/7 2026-02-20T21:04:27.287 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 3/7 2026-02-20T21:04:27.287 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 4/7 2026-02-20T21:04:27.287 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 5/7 2026-02-20T21:04:27.287 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 6/7 2026-02-20T21:04:27.344 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 7/7 2026-02-20T21:04:27.344 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:27.345 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:27.345 INFO:teuthology.orchestra.run.trial163.stdout: ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:27.345 INFO:teuthology.orchestra.run.trial163.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-20T21:04:27.345 INFO:teuthology.orchestra.run.trial163.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-20T21:04:27.345 INFO:teuthology.orchestra.run.trial163.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-20T21:04:27.345 INFO:teuthology.orchestra.run.trial163.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-20T21:04:27.345 INFO:teuthology.orchestra.run.trial163.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-20T21:04:27.345 INFO:teuthology.orchestra.run.trial163.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-20T21:04:27.345 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:27.345 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:27.375 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install ceph-fuse 2026-02-20T21:04:27.416 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:00:49 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:27.498 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:27.499 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:27.499 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repository Size 2026-02-20T21:04:27.499 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:27.499 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:27.499 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mgr-dashboard noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 11 M 2026-02-20T21:04:27.499 INFO:teuthology.orchestra.run.trial069.stdout:Installing dependencies: 2026-02-20T21:04:27.499 INFO:teuthology.orchestra.run.trial069.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: ceph-grafana-dashboards noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 43 k 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: ceph-prometheus-alerts noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 17 k 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: python3-jmespath noarch 1.0.1-1.el9 appstream 48 k 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:27.500 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:27.501 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:27.501 INFO:teuthology.orchestra.run.trial069.stdout:Install 14 Packages 2026-02-20T21:04:27.501 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:27.501 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 16 M 2026-02-20T21:04:27.501 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 158 M 2026-02-20T21:04:27.501 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:27.686 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:43 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:27.771 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout: Package Architecture Version Repository Size 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout: ceph-fuse x86_64 2:20.2.0-677.gf7870454.el9 ceph 943 k 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout:Install 1 Package 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 943 k 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 2.7 M 2026-02-20T21:04:27.772 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:27.891 INFO:teuthology.orchestra.run.trial163.stdout:ceph-fuse-20.2.0-677.gf7870454.el9.x86_64.rpm 7.7 MB/s | 943 kB 00:00 2026-02-20T21:04:27.892 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:27.892 INFO:teuthology.orchestra.run.trial163.stdout:Total 7.7 MB/s | 943 kB 00:00 2026-02-20T21:04:27.892 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:27.896 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:27.896 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:27.930 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:27.930 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:28.020 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:28.038 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:28.253 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:28.310 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:28.310 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:28.310 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:28.311 INFO:teuthology.orchestra.run.trial163.stdout: ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:28.311 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:28.311 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:28.326 INFO:teuthology.orchestra.run.trial069.stdout:(1/14): ceph-prometheus-alerts-20.2.0-677.gf787 277 kB/s | 17 kB 00:00 2026-02-20T21:04:28.338 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install ceph-volume 2026-02-20T21:04:28.339 INFO:teuthology.orchestra.run.trial069.stdout:(2/14): ceph-grafana-dashboards-20.2.0-677.gf78 570 kB/s | 43 kB 00:00 2026-02-20T21:04:28.466 INFO:teuthology.orchestra.run.trial069.stdout:(3/14): python3-jmespath-1.0.1-1.el9.noarch.rpm 375 kB/s | 48 kB 00:00 2026-02-20T21:04:28.558 INFO:teuthology.orchestra.run.trial069.stdout:(4/14): ceph-mgr-dashboard-20.2.0-677.gf7870454 36 MB/s | 11 MB 00:00 2026-02-20T21:04:28.560 INFO:teuthology.orchestra.run.trial069.stdout:(5/14): protobuf-3.14.0-17.el9.x86_64.rpm 4.3 MB/s | 1.0 MB 00:00 2026-02-20T21:04:28.604 INFO:teuthology.orchestra.run.trial069.stdout:(6/14): python3-protobuf-3.14.0-17.el9.noarch.r 1.9 MB/s | 267 kB 00:00 2026-02-20T21:04:28.624 INFO:teuthology.orchestra.run.trial069.stdout:(7/14): protobuf-compiler-3.14.0-17.el9.x86_64. 13 MB/s | 862 kB 00:00 2026-02-20T21:04:28.642 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:44 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:28.655 INFO:teuthology.orchestra.run.trial069.stdout:(8/14): grpc-data-1.46.7-10.el9.noarch.rpm 385 kB/s | 19 kB 00:00 2026-02-20T21:04:28.684 INFO:teuthology.orchestra.run.trial069.stdout:(9/14): abseil-cpp-20211102.0-4.el9.x86_64.rpm 4.4 MB/s | 551 kB 00:00 2026-02-20T21:04:28.701 INFO:teuthology.orchestra.run.trial069.stdout:(10/14): python3-repoze-lru-0.7-16.el9.noarch.r 1.8 MB/s | 31 kB 00:00 2026-02-20T21:04:28.714 INFO:teuthology.orchestra.run.trial069.stdout:(11/14): python3-grpcio-tools-1.46.7-10.el9.x86 2.4 MB/s | 144 kB 00:00 2026-02-20T21:04:28.720 INFO:teuthology.orchestra.run.trial069.stdout:(12/14): python3-routes-2.5.1-5.el9.noarch.rpm 9.5 MB/s | 188 kB 00:00 2026-02-20T21:04:28.725 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:28.726 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:28.726 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repository Size 2026-02-20T21:04:28.726 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:28.726 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:28.726 INFO:teuthology.orchestra.run.trial163.stdout: ceph-volume noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 297 k 2026-02-20T21:04:28.726 INFO:teuthology.orchestra.run.trial163.stdout:Installing dependencies: 2026-02-20T21:04:28.726 INFO:teuthology.orchestra.run.trial163.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-20T21:04:28.727 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:28.727 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:28.727 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:28.727 INFO:teuthology.orchestra.run.trial163.stdout:Install 2 Packages 2026-02-20T21:04:28.727 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:28.727 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 648 k 2026-02-20T21:04:28.727 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 2.2 M 2026-02-20T21:04:28.727 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:28.732 INFO:teuthology.orchestra.run.trial069.stdout:(13/14): python3-xmltodict-0.12.0-15.el9.noarch 1.2 MB/s | 22 kB 00:00 2026-02-20T21:04:28.781 INFO:teuthology.orchestra.run.trial069.stdout:(14/14): python3-grpcio-1.46.7-10.el9.x86_64.rp 13 MB/s | 2.0 MB 00:00 2026-02-20T21:04:28.782 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:28.782 INFO:teuthology.orchestra.run.trial069.stdout:Total 12 MB/s | 16 MB 00:01 2026-02-20T21:04:28.818 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:28.828 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:28.828 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:28.878 INFO:teuthology.orchestra.run.trial163.stdout:(1/2): cryptsetup-2.8.1-3.el9.x86_64.rpm 3.9 MB/s | 351 kB 00:00 2026-02-20T21:04:28.886 INFO:teuthology.orchestra.run.trial163.stdout:(2/2): ceph-volume-20.2.0-677.gf7870454.el9.noa 3.0 MB/s | 297 kB 00:00 2026-02-20T21:04:28.888 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:28.888 INFO:teuthology.orchestra.run.trial163.stdout:Total 3.9 MB/s | 648 kB 00:00 2026-02-20T21:04:28.897 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:28.904 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:28.904 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:28.922 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:28.922 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:28.947 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:28.947 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:29.028 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:29.037 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:29.046 INFO:teuthology.orchestra.run.trial069.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 1/14 2026-02-20T21:04:29.049 INFO:teuthology.orchestra.run.trial069.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 2/14 2026-02-20T21:04:29.054 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 3/14 2026-02-20T21:04:29.065 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 4/14 2026-02-20T21:04:29.067 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 5/14 2026-02-20T21:04:29.089 INFO:teuthology.orchestra.run.trial163.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 1/2 2026-02-20T21:04:29.092 INFO:teuthology.orchestra.run.trial163.stdout: Installing : ceph-volume-2:20.2.0-677.gf7870454.el9.noarch 2/2 2026-02-20T21:04:29.095 INFO:teuthology.orchestra.run.trial069.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 6/14 2026-02-20T21:04:29.107 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: ceph-volume-2:20.2.0-677.gf7870454.el9.noarch 2/2 2026-02-20T21:04:29.107 INFO:teuthology.orchestra.run.trial163.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:29.107 INFO:teuthology.orchestra.run.trial163.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-20T21:04:29.107 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:29.139 INFO:teuthology.orchestra.run.trial069.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 7/14 2026-02-20T21:04:29.169 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 8/14 2026-02-20T21:04:29.180 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 9/14 2026-02-20T21:04:29.187 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 10/14 2026-02-20T21:04:29.189 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-jmespath-1.0.1-1.el9.noarch 11/14 2026-02-20T21:04:29.196 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-prometheus-alerts-2:20.2.0-677.gf7870454.el9. 12/14 2026-02-20T21:04:29.328 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : ceph-volume-2:20.2.0-677.gf7870454.el9.noarch 1/2 2026-02-20T21:04:29.385 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 2/2 2026-02-20T21:04:29.385 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:29.385 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:29.386 INFO:teuthology.orchestra.run.trial163.stdout: ceph-volume-2:20.2.0-677.gf7870454.el9.noarch cryptsetup-2.8.1-3.el9.x86_64 2026-02-20T21:04:29.386 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:29.386 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:29.414 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install librados-devel 2026-02-20T21:04:29.720 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:45 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:29.803 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:29.804 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:29.804 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repo Size 2026-02-20T21:04:29.804 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:29.804 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:29.804 INFO:teuthology.orchestra.run.trial163.stdout: librados-devel x86_64 2:20.2.0-677.gf7870454.el9 ceph 126 k 2026-02-20T21:04:29.804 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:29.804 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:29.804 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:29.804 INFO:teuthology.orchestra.run.trial163.stdout:Install 1 Package 2026-02-20T21:04:29.805 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:29.805 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 126 k 2026-02-20T21:04:29.805 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 449 k 2026-02-20T21:04:29.805 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:29.889 INFO:teuthology.orchestra.run.trial163.stdout:librados-devel-20.2.0-677.gf7870454.el9.x86_64. 1.4 MB/s | 126 kB 00:00 2026-02-20T21:04:29.890 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:29.890 INFO:teuthology.orchestra.run.trial163.stdout:Total 1.4 MB/s | 126 kB 00:00 2026-02-20T21:04:29.890 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:29.893 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:29.893 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:29.923 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:29.923 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:29.977 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:29.993 INFO:teuthology.orchestra.run.trial163.stdout: Installing : librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:30.072 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:30.129 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:30.129 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:30.129 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:30.129 INFO:teuthology.orchestra.run.trial163.stdout: librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:30.129 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:30.130 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:30.158 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install libcephfs2 2026-02-20T21:04:30.334 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-grafana-dashboards-2:20.2.0-677.gf7870454.el9 13/14 2026-02-20T21:04:30.339 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noar 14/14 2026-02-20T21:04:30.466 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:46 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:30.531 INFO:teuthology.orchestra.run.trial163.stdout:Package libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:30.534 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noar 14/14 2026-02-20T21:04:30.534 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-grafana-dashboards-2:20.2.0-677.gf7870454.el9 1/14 2026-02-20T21:04:30.534 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noar 2/14 2026-02-20T21:04:30.534 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-prometheus-alerts-2:20.2.0-677.gf7870454.el9. 3/14 2026-02-20T21:04:30.535 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 4/14 2026-02-20T21:04:30.535 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-jmespath-1.0.1-1.el9.noarch 5/14 2026-02-20T21:04:30.535 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 6/14 2026-02-20T21:04:30.535 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 7/14 2026-02-20T21:04:30.535 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 8/14 2026-02-20T21:04:30.535 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 9/14 2026-02-20T21:04:30.535 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 10/14 2026-02-20T21:04:30.535 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 11/14 2026-02-20T21:04:30.535 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 12/14 2026-02-20T21:04:30.535 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 13/14 2026-02-20T21:04:30.549 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:30.550 INFO:teuthology.orchestra.run.trial163.stdout:Nothing to do. 2026-02-20T21:04:30.550 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:30.572 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install libcephfs-devel 2026-02-20T21:04:30.588 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 14/14 2026-02-20T21:04:30.588 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:30.588 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:30.588 INFO:teuthology.orchestra.run.trial069.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-20T21:04:30.589 INFO:teuthology.orchestra.run.trial069.stdout: ceph-grafana-dashboards-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:30.589 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mgr-dashboard-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:30.589 INFO:teuthology.orchestra.run.trial069.stdout: ceph-prometheus-alerts-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:30.589 INFO:teuthology.orchestra.run.trial069.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-20T21:04:30.589 INFO:teuthology.orchestra.run.trial069.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-20T21:04:30.589 INFO:teuthology.orchestra.run.trial069.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-20T21:04:30.589 INFO:teuthology.orchestra.run.trial069.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-20T21:04:30.589 INFO:teuthology.orchestra.run.trial069.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-20T21:04:30.589 INFO:teuthology.orchestra.run.trial069.stdout: python3-jmespath-1.0.1-1.el9.noarch 2026-02-20T21:04:30.589 INFO:teuthology.orchestra.run.trial069.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-20T21:04:30.589 INFO:teuthology.orchestra.run.trial069.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-20T21:04:30.590 INFO:teuthology.orchestra.run.trial069.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-20T21:04:30.590 INFO:teuthology.orchestra.run.trial069.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-20T21:04:30.590 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:30.590 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:30.620 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-20T21:04:30.882 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:46 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:30.922 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:00:52 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:30.966 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:30.967 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:30.967 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repo Size 2026-02-20T21:04:30.967 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:30.967 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:30.968 INFO:teuthology.orchestra.run.trial163.stdout: libcephfs-devel x86_64 2:20.2.0-677.gf7870454.el9 ceph 34 k 2026-02-20T21:04:30.968 INFO:teuthology.orchestra.run.trial163.stdout:Installing dependencies: 2026-02-20T21:04:30.968 INFO:teuthology.orchestra.run.trial163.stdout: libcephfs-proxy2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 24 k 2026-02-20T21:04:30.968 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:30.968 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:30.968 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:30.968 INFO:teuthology.orchestra.run.trial163.stdout:Install 2 Packages 2026-02-20T21:04:30.968 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:30.968 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 58 k 2026-02-20T21:04:30.968 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 207 k 2026-02-20T21:04:30.968 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:31.005 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:31.005 INFO:teuthology.orchestra.run.trial069.stdout:====================================================================================== 2026-02-20T21:04:31.005 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repository Size 2026-02-20T21:04:31.005 INFO:teuthology.orchestra.run.trial069.stdout:====================================================================================== 2026-02-20T21:04:31.005 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:31.005 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mgr-diskprediction-local noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 7.4 M 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout:Installing dependencies: 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout:====================================================================================== 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout:Install 12 Packages 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:31.006 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 43 M 2026-02-20T21:04:31.007 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 234 M 2026-02-20T21:04:31.007 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:31.032 INFO:teuthology.orchestra.run.trial163.stdout:(1/2): libcephfs-devel-20.2.0-677.gf7870454.el9 532 kB/s | 34 kB 00:00 2026-02-20T21:04:31.033 INFO:teuthology.orchestra.run.trial163.stdout:(2/2): libcephfs-proxy2-20.2.0-677.gf7870454.el 366 kB/s | 24 kB 00:00 2026-02-20T21:04:31.035 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:31.035 INFO:teuthology.orchestra.run.trial163.stdout:Total 859 kB/s | 58 kB 00:00 2026-02-20T21:04:31.035 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:31.038 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:31.038 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:31.069 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:31.069 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:31.117 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:31.118 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 1/2 2026-02-20T21:04:31.132 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 1/2 2026-02-20T21:04:31.148 INFO:teuthology.orchestra.run.trial163.stdout: Installing : libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 2/2 2026-02-20T21:04:31.169 INFO:teuthology.orchestra.run.trial069.stdout:(1/12): libquadmath-11.5.0-14.el9.x86_64.rpm 4.0 MB/s | 184 kB 00:00 2026-02-20T21:04:31.187 INFO:teuthology.orchestra.run.trial069.stdout:(2/12): libgfortran-11.5.0-14.el9.x86_64.rpm 13 MB/s | 794 kB 00:00 2026-02-20T21:04:31.207 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 2/2 2026-02-20T21:04:31.208 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/2 2026-02-20T21:04:31.264 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 2/2 2026-02-20T21:04:31.264 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:31.264 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:31.265 INFO:teuthology.orchestra.run.trial163.stdout: libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:31.265 INFO:teuthology.orchestra.run.trial163.stdout: libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:31.265 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:31.265 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:31.278 INFO:teuthology.orchestra.run.trial069.stdout:(3/12): flexiblas-3.0.4-9.el9.x86_64.rpm 273 kB/s | 30 kB 00:00 2026-02-20T21:04:31.292 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install librados2 2026-02-20T21:04:31.310 INFO:teuthology.orchestra.run.trial069.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-9.el9.x 471 kB/s | 15 kB 00:00 2026-02-20T21:04:31.353 INFO:teuthology.orchestra.run.trial069.stdout:(5/12): ceph-mgr-diskprediction-local-20.2.0-67 32 MB/s | 7.4 MB 00:00 2026-02-20T21:04:31.355 INFO:teuthology.orchestra.run.trial069.stdout:(6/12): openblas-0.3.29-1.el9.x86_64.rpm 926 kB/s | 42 kB 00:00 2026-02-20T21:04:31.442 INFO:teuthology.orchestra.run.trial069.stdout:(7/12): python3-devel-3.9.25-3.el9.x86_64.rpm 2.7 MB/s | 244 kB 00:00 2026-02-20T21:04:31.463 INFO:teuthology.orchestra.run.trial069.stdout:(8/12): flexiblas-netlib-3.0.4-9.el9.x86_64.rpm 11 MB/s | 3.0 MB 00:00 2026-02-20T21:04:31.492 INFO:teuthology.orchestra.run.trial069.stdout:(9/12): python3-numpy-f2py-1.23.5-2.el9.x86_64. 15 MB/s | 442 kB 00:00 2026-02-20T21:04:31.594 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:47 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:31.622 INFO:teuthology.orchestra.run.trial069.stdout:(10/12): openblas-openmp-0.3.29-1.el9.x86_64.rp 20 MB/s | 5.3 MB 00:00 2026-02-20T21:04:31.641 INFO:teuthology.orchestra.run.trial069.stdout:(11/12): python3-numpy-1.23.5-2.el9.x86_64.rpm 31 MB/s | 6.1 MB 00:00 2026-02-20T21:04:31.659 INFO:teuthology.orchestra.run.trial163.stdout:Package librados2-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:31.673 INFO:teuthology.orchestra.run.trial069.stdout:(12/12): python3-scipy-1.9.3-2.el9.x86_64.rpm 106 MB/s | 19 MB 00:00 2026-02-20T21:04:31.673 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:31.674 INFO:teuthology.orchestra.run.trial069.stdout:Total 64 MB/s | 43 MB 00:00 2026-02-20T21:04:31.677 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:31.678 INFO:teuthology.orchestra.run.trial163.stdout:Nothing to do. 2026-02-20T21:04:31.678 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:31.700 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install librbd1 2026-02-20T21:04:31.832 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:31.845 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:31.846 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:32.007 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:48 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:32.047 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:32.047 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:32.073 INFO:teuthology.orchestra.run.trial163.stdout:Package librbd1-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:32.091 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:32.092 INFO:teuthology.orchestra.run.trial163.stdout:Nothing to do. 2026-02-20T21:04:32.092 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:32.114 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install python3-rados 2026-02-20T21:04:32.264 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:32.267 INFO:teuthology.orchestra.run.trial069.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 1/12 2026-02-20T21:04:32.272 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 2/12 2026-02-20T21:04:32.306 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 3/12 2026-02-20T21:04:32.311 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 4/12 2026-02-20T21:04:32.370 INFO:teuthology.orchestra.run.trial069.stdout: Installing : openblas-0.3.29-1.el9.x86_64 5/12 2026-02-20T21:04:32.373 INFO:teuthology.orchestra.run.trial069.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 6/12 2026-02-20T21:04:32.394 INFO:teuthology.orchestra.run.trial069.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 7/12 2026-02-20T21:04:32.417 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:48 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:32.482 INFO:teuthology.orchestra.run.trial163.stdout:Package python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:32.500 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:32.501 INFO:teuthology.orchestra.run.trial163.stdout:Nothing to do. 2026-02-20T21:04:32.501 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:32.523 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install python3-rgw 2026-02-20T21:04:32.712 INFO:teuthology.orchestra.run.trial069.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 8/12 2026-02-20T21:04:32.783 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 9/12 2026-02-20T21:04:32.824 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:48 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:32.889 INFO:teuthology.orchestra.run.trial163.stdout:Package python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:32.907 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:32.908 INFO:teuthology.orchestra.run.trial163.stdout:Nothing to do. 2026-02-20T21:04:32.908 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:32.930 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install python3-cephfs 2026-02-20T21:04:33.238 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:49 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:33.304 INFO:teuthology.orchestra.run.trial163.stdout:Package python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:33.322 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:33.323 INFO:teuthology.orchestra.run.trial163.stdout:Nothing to do. 2026-02-20T21:04:33.323 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:33.345 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install python3-rbd 2026-02-20T21:04:33.444 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 10/12 2026-02-20T21:04:33.653 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:49 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:33.719 INFO:teuthology.orchestra.run.trial163.stdout:Package python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:33.737 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:33.737 INFO:teuthology.orchestra.run.trial163.stdout:Nothing to do. 2026-02-20T21:04:33.738 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:33.759 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install rbd-fuse 2026-02-20T21:04:33.896 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 11/12 2026-02-20T21:04:33.896 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-mgr-diskprediction-local-2:20.2.0-677.gf78704 12/12 2026-02-20T21:04:34.068 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:50 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:34.151 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:34.152 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:34.152 INFO:teuthology.orchestra.run.trial163.stdout: Package Architecture Version Repository Size 2026-02-20T21:04:34.152 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:34.152 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:34.152 INFO:teuthology.orchestra.run.trial163.stdout: rbd-fuse x86_64 2:20.2.0-677.gf7870454.el9 ceph 91 k 2026-02-20T21:04:34.152 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:34.152 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:34.152 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:34.152 INFO:teuthology.orchestra.run.trial163.stdout:Install 1 Package 2026-02-20T21:04:34.152 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:34.153 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 91 k 2026-02-20T21:04:34.153 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 238 k 2026-02-20T21:04:34.153 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:34.193 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:20.2.0-677.gf78704 12/12 2026-02-20T21:04:34.193 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-mgr-diskprediction-local-2:20.2.0-677.gf78704 1/12 2026-02-20T21:04:34.193 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 2/12 2026-02-20T21:04:34.193 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 3/12 2026-02-20T21:04:34.193 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 4/12 2026-02-20T21:04:34.193 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 5/12 2026-02-20T21:04:34.193 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 6/12 2026-02-20T21:04:34.193 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 7/12 2026-02-20T21:04:34.194 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 8/12 2026-02-20T21:04:34.194 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 9/12 2026-02-20T21:04:34.194 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 10/12 2026-02-20T21:04:34.194 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 11/12 2026-02-20T21:04:34.228 INFO:teuthology.orchestra.run.trial163.stdout:rbd-fuse-20.2.0-677.gf7870454.el9.x86_64.rpm 1.2 MB/s | 91 kB 00:00 2026-02-20T21:04:34.229 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:34.229 INFO:teuthology.orchestra.run.trial163.stdout:Total 1.1 MB/s | 91 kB 00:00 2026-02-20T21:04:34.229 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:34.233 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:34.233 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:34.249 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 12/12 2026-02-20T21:04:34.249 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:34.249 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:34.249 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mgr-diskprediction-local-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:34.249 INFO:teuthology.orchestra.run.trial069.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-20T21:04:34.250 INFO:teuthology.orchestra.run.trial069.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-20T21:04:34.250 INFO:teuthology.orchestra.run.trial069.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-20T21:04:34.250 INFO:teuthology.orchestra.run.trial069.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-20T21:04:34.250 INFO:teuthology.orchestra.run.trial069.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-20T21:04:34.250 INFO:teuthology.orchestra.run.trial069.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-20T21:04:34.250 INFO:teuthology.orchestra.run.trial069.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-20T21:04:34.250 INFO:teuthology.orchestra.run.trial069.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-20T21:04:34.250 INFO:teuthology.orchestra.run.trial069.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-20T21:04:34.250 INFO:teuthology.orchestra.run.trial069.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-20T21:04:34.250 INFO:teuthology.orchestra.run.trial069.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-20T21:04:34.250 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:34.251 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:34.264 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:34.264 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:34.290 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install ceph-mgr-rook 2026-02-20T21:04:34.313 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:34.330 INFO:teuthology.orchestra.run.trial163.stdout: Installing : rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:34.408 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:34.465 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:34.465 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:34.466 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:34.466 INFO:teuthology.orchestra.run.trial163.stdout: rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:34.466 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:34.466 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:34.494 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install rbd-mirror 2026-02-20T21:04:34.599 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:00:56 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:34.683 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:34.684 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:34.684 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repository Size 2026-02-20T21:04:34.684 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:34.684 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:34.684 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mgr-rook noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 50 k 2026-02-20T21:04:34.684 INFO:teuthology.orchestra.run.trial069.stdout:Installing dependencies: 2026-02-20T21:04:34.684 INFO:teuthology.orchestra.run.trial069.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-20T21:04:34.685 INFO:teuthology.orchestra.run.trial069.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-20T21:04:34.685 INFO:teuthology.orchestra.run.trial069.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-20T21:04:34.685 INFO:teuthology.orchestra.run.trial069.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-20T21:04:34.685 INFO:teuthology.orchestra.run.trial069.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-20T21:04:34.685 INFO:teuthology.orchestra.run.trial069.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-20T21:04:34.685 INFO:teuthology.orchestra.run.trial069.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-20T21:04:34.685 INFO:teuthology.orchestra.run.trial069.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-20T21:04:34.685 INFO:teuthology.orchestra.run.trial069.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-20T21:04:34.685 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:34.685 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:34.685 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:34.685 INFO:teuthology.orchestra.run.trial069.stdout:Install 10 Packages 2026-02-20T21:04:34.686 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:34.686 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 2.0 M 2026-02-20T21:04:34.686 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 25 M 2026-02-20T21:04:34.686 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:34.805 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:50 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:34.888 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:34.888 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:34.888 INFO:teuthology.orchestra.run.trial163.stdout: Package Arch Version Repo Size 2026-02-20T21:04:34.888 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:34.888 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:34.888 INFO:teuthology.orchestra.run.trial163.stdout: rbd-mirror x86_64 2:20.2.0-677.gf7870454.el9 ceph 2.9 M 2026-02-20T21:04:34.888 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:34.888 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:34.888 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:34.888 INFO:teuthology.orchestra.run.trial163.stdout:Install 1 Package 2026-02-20T21:04:34.888 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:34.888 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 2.9 M 2026-02-20T21:04:34.889 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 11 M 2026-02-20T21:04:34.889 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:34.923 INFO:teuthology.orchestra.run.trial069.stdout:(1/10): ceph-mgr-rook-20.2.0-677.gf7870454.el9. 690 kB/s | 50 kB 00:00 2026-02-20T21:04:35.036 INFO:teuthology.orchestra.run.trial163.stdout:rbd-mirror-20.2.0-677.gf7870454.el9.x86_64.rpm 20 MB/s | 2.9 MB 00:00 2026-02-20T21:04:35.037 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:35.037 INFO:teuthology.orchestra.run.trial163.stdout:Total 20 MB/s | 2.9 MB 00:00 2026-02-20T21:04:35.037 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:35.041 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:35.042 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:35.080 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:35.080 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:35.258 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:35.260 INFO:teuthology.orchestra.run.trial163.stdout: Installing : rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:35.284 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:35.284 INFO:teuthology.orchestra.run.trial163.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:35.284 INFO:teuthology.orchestra.run.trial163.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-20T21:04:35.284 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-20T21:04:35.284 INFO:teuthology.orchestra.run.trial163.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-20T21:04:35.285 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:35.555 INFO:teuthology.orchestra.run.trial069.stdout:(2/10): python3-requests-oauthlib-1.3.0-12.el9. 85 kB/s | 54 kB 00:00 2026-02-20T21:04:35.597 INFO:teuthology.orchestra.run.trial069.stdout:(3/10): python3-cachetools-4.2.4-1.el9.noarch.r 778 kB/s | 32 kB 00:00 2026-02-20T21:04:35.606 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:35.606 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:35.606 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:35.606 INFO:teuthology.orchestra.run.trial163.stdout: rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:35.606 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:35.606 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:35.608 INFO:teuthology.orchestra.run.trial069.stdout:(4/10): python3-certifi-2023.05.07-4.el9.noarch 1.2 MB/s | 14 kB 00:00 2026-02-20T21:04:35.635 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install rbd-nbd 2026-02-20T21:04:35.650 INFO:teuthology.orchestra.run.trial069.stdout:(5/10): python3-google-auth-2.45.0-1.el9.noarch 5.9 MB/s | 254 kB 00:00 2026-02-20T21:04:35.691 INFO:teuthology.orchestra.run.trial069.stdout:(6/10): python3-pyasn1-0.4.8-6.el9.noarch.rpm 189 kB/s | 159 kB 00:00 2026-02-20T21:04:35.709 INFO:teuthology.orchestra.run.trial069.stdout:(7/10): python3-kubernetes-26.1.0-3.el9.noarch. 18 MB/s | 1.0 MB 00:00 2026-02-20T21:04:35.721 INFO:teuthology.orchestra.run.trial069.stdout:(8/10): python3-websocket-client-1.2.3-2.el9.no 7.1 MB/s | 90 kB 00:00 2026-02-20T21:04:35.732 INFO:teuthology.orchestra.run.trial069.stdout:(9/10): python3-rsa-4.9-2.el9.noarch.rpm 1.4 MB/s | 59 kB 00:00 2026-02-20T21:04:35.841 INFO:teuthology.orchestra.run.trial069.stdout:(10/10): python3-pyasn1-modules-0.4.8-6.el9.noa 281 kB/s | 279 kB 00:00 2026-02-20T21:04:35.843 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:35.843 INFO:teuthology.orchestra.run.trial069.stdout:Total 1.7 MB/s | 2.0 MB 00:01 2026-02-20T21:04:35.869 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:35.878 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:35.878 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:35.917 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:35.917 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:35.948 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:51 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:35.992 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:36.032 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:36.033 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:36.033 INFO:teuthology.orchestra.run.trial163.stdout: Package Architecture Version Repository Size 2026-02-20T21:04:36.033 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:36.033 INFO:teuthology.orchestra.run.trial163.stdout:Installing: 2026-02-20T21:04:36.033 INFO:teuthology.orchestra.run.trial163.stdout: rbd-nbd x86_64 2:20.2.0-677.gf7870454.el9 ceph 180 k 2026-02-20T21:04:36.033 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:36.033 INFO:teuthology.orchestra.run.trial163.stdout:Transaction Summary 2026-02-20T21:04:36.034 INFO:teuthology.orchestra.run.trial163.stdout:================================================================================ 2026-02-20T21:04:36.034 INFO:teuthology.orchestra.run.trial163.stdout:Install 1 Package 2026-02-20T21:04:36.034 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:36.034 INFO:teuthology.orchestra.run.trial163.stdout:Total download size: 180 k 2026-02-20T21:04:36.034 INFO:teuthology.orchestra.run.trial163.stdout:Installed size: 498 k 2026-02-20T21:04:36.034 INFO:teuthology.orchestra.run.trial163.stdout:Downloading Packages: 2026-02-20T21:04:36.043 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/10 2026-02-20T21:04:36.056 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/10 2026-02-20T21:04:36.071 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/10 2026-02-20T21:04:36.075 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/10 2026-02-20T21:04:36.083 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/10 2026-02-20T21:04:36.120 INFO:teuthology.orchestra.run.trial163.stdout:rbd-nbd-20.2.0-677.gf7870454.el9.x86_64.rpm 2.0 MB/s | 180 kB 00:00 2026-02-20T21:04:36.120 INFO:teuthology.orchestra.run.trial163.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:36.121 INFO:teuthology.orchestra.run.trial163.stdout:Total 2.0 MB/s | 180 kB 00:00 2026-02-20T21:04:36.121 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction check 2026-02-20T21:04:36.123 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/10 2026-02-20T21:04:36.125 INFO:teuthology.orchestra.run.trial163.stdout:Transaction check succeeded. 2026-02-20T21:04:36.125 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction test 2026-02-20T21:04:36.135 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 7/10 2026-02-20T21:04:36.155 INFO:teuthology.orchestra.run.trial163.stdout:Transaction test succeeded. 2026-02-20T21:04:36.155 INFO:teuthology.orchestra.run.trial163.stdout:Running transaction 2026-02-20T21:04:36.210 INFO:teuthology.orchestra.run.trial163.stdout: Preparing : 1/1 2026-02-20T21:04:36.226 INFO:teuthology.orchestra.run.trial163.stdout: Installing : rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:36.305 INFO:teuthology.orchestra.run.trial163.stdout: Running scriptlet: rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:36.362 INFO:teuthology.orchestra.run.trial163.stdout: Verifying : rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:36.362 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:36.362 INFO:teuthology.orchestra.run.trial163.stdout:Installed: 2026-02-20T21:04:36.362 INFO:teuthology.orchestra.run.trial163.stdout: rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:36.362 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:36.362 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:36.390 DEBUG:teuthology.orchestra.run.trial163:> sudo yum -y install rbd-mirror 2026-02-20T21:04:36.456 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 8/10 2026-02-20T21:04:36.471 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 9/10 2026-02-20T21:04:36.472 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 10/10 2026-02-20T21:04:36.580 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 10/10 2026-02-20T21:04:36.580 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 1/10 2026-02-20T21:04:36.581 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 2/10 2026-02-20T21:04:36.581 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 3/10 2026-02-20T21:04:36.581 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 4/10 2026-02-20T21:04:36.581 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 5/10 2026-02-20T21:04:36.581 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 6/10 2026-02-20T21:04:36.581 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 7/10 2026-02-20T21:04:36.581 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 8/10 2026-02-20T21:04:36.581 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 9/10 2026-02-20T21:04:36.637 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 10/10 2026-02-20T21:04:36.637 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:36.637 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:36.637 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mgr-rook-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:36.637 INFO:teuthology.orchestra.run.trial069.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-20T21:04:36.637 INFO:teuthology.orchestra.run.trial069.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-20T21:04:36.638 INFO:teuthology.orchestra.run.trial069.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-20T21:04:36.638 INFO:teuthology.orchestra.run.trial069.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-20T21:04:36.638 INFO:teuthology.orchestra.run.trial069.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-20T21:04:36.638 INFO:teuthology.orchestra.run.trial069.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-20T21:04:36.638 INFO:teuthology.orchestra.run.trial069.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-20T21:04:36.638 INFO:teuthology.orchestra.run.trial069.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-20T21:04:36.638 INFO:teuthology.orchestra.run.trial069.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-20T21:04:36.638 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:36.638 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:36.667 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install ceph-mgr-cephadm 2026-02-20T21:04:36.701 INFO:teuthology.orchestra.run.trial163.stdout:Last metadata expiration check: 0:00:52 ago on Fri 20 Feb 2026 09:03:44 PM UTC. 2026-02-20T21:04:36.767 INFO:teuthology.orchestra.run.trial163.stdout:Package rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:36.785 INFO:teuthology.orchestra.run.trial163.stdout:Dependencies resolved. 2026-02-20T21:04:36.786 INFO:teuthology.orchestra.run.trial163.stdout:Nothing to do. 2026-02-20T21:04:36.786 INFO:teuthology.orchestra.run.trial163.stdout:Complete! 2026-02-20T21:04:36.808 DEBUG:teuthology.parallel:result is None 2026-02-20T21:04:36.973 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:00:59 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:37.056 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:37.057 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:37.057 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repository Size 2026-02-20T21:04:37.057 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:37.057 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:37.057 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mgr-cephadm noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 173 k 2026-02-20T21:04:37.058 INFO:teuthology.orchestra.run.trial069.stdout:Installing dependencies: 2026-02-20T21:04:37.058 INFO:teuthology.orchestra.run.trial069.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-20T21:04:37.058 INFO:teuthology.orchestra.run.trial069.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-20T21:04:37.058 INFO:teuthology.orchestra.run.trial069.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-20T21:04:37.058 INFO:teuthology.orchestra.run.trial069.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-20T21:04:37.058 INFO:teuthology.orchestra.run.trial069.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-20T21:04:37.058 INFO:teuthology.orchestra.run.trial069.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-20T21:04:37.058 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:37.058 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:37.058 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:37.058 INFO:teuthology.orchestra.run.trial069.stdout:Install 7 Packages 2026-02-20T21:04:37.058 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:37.059 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 7.1 M 2026-02-20T21:04:37.059 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 33 M 2026-02-20T21:04:37.059 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:37.225 INFO:teuthology.orchestra.run.trial069.stdout:(1/7): python3-jinja2-2.11.3-8.el9.noarch.rpm 7.3 MB/s | 249 kB 00:00 2026-02-20T21:04:37.230 INFO:teuthology.orchestra.run.trial069.stdout:(2/7): python3-markupsafe-1.1.1-12.el9.x86_64.r 6.9 MB/s | 35 kB 00:00 2026-02-20T21:04:37.278 INFO:teuthology.orchestra.run.trial069.stdout:(3/7): ceph-mgr-cephadm-20.2.0-677.gf7870454.el 2.0 MB/s | 173 kB 00:00 2026-02-20T21:04:37.285 INFO:teuthology.orchestra.run.trial069.stdout:(4/7): python3-babel-2.9.1-2.el9.noarch.rpm 64 MB/s | 6.0 MB 00:00 2026-02-20T21:04:37.345 INFO:teuthology.orchestra.run.trial069.stdout:(5/7): python3-natsort-7.1.1-5.el9.noarch.rpm 854 kB/s | 58 kB 00:00 2026-02-20T21:04:37.362 INFO:teuthology.orchestra.run.trial069.stdout:(6/7): python3-typing-extensions-4.15.0-1.el9.n 1.1 MB/s | 86 kB 00:00 2026-02-20T21:04:37.367 INFO:teuthology.orchestra.run.trial069.stdout:(7/7): python3-asyncssh-2.13.2-5.el9.noarch.rpm 3.9 MB/s | 548 kB 00:00 2026-02-20T21:04:37.367 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:37.367 INFO:teuthology.orchestra.run.trial069.stdout:Total 23 MB/s | 7.1 MB 00:00 2026-02-20T21:04:37.406 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:37.412 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:37.413 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:37.483 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:37.483 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:37.572 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:37.621 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 1/7 2026-02-20T21:04:37.631 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/7 2026-02-20T21:04:37.636 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/7 2026-02-20T21:04:37.855 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 4/7 2026-02-20T21:04:37.879 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 5/7 2026-02-20T21:04:37.902 INFO:teuthology.orchestra.run.trial069.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 6/7 2026-02-20T21:04:37.904 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 7/7 2026-02-20T21:04:38.014 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 7/7 2026-02-20T21:04:38.014 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 1/7 2026-02-20T21:04:38.014 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/7 2026-02-20T21:04:38.014 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 3/7 2026-02-20T21:04:38.015 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 4/7 2026-02-20T21:04:38.015 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 5/7 2026-02-20T21:04:38.015 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 6/7 2026-02-20T21:04:38.069 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 7/7 2026-02-20T21:04:38.069 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:38.069 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:38.069 INFO:teuthology.orchestra.run.trial069.stdout: ceph-mgr-cephadm-2:20.2.0-677.gf7870454.el9.noarch 2026-02-20T21:04:38.070 INFO:teuthology.orchestra.run.trial069.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-20T21:04:38.070 INFO:teuthology.orchestra.run.trial069.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-20T21:04:38.070 INFO:teuthology.orchestra.run.trial069.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-20T21:04:38.070 INFO:teuthology.orchestra.run.trial069.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-20T21:04:38.070 INFO:teuthology.orchestra.run.trial069.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-20T21:04:38.070 INFO:teuthology.orchestra.run.trial069.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-20T21:04:38.070 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:38.070 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:38.099 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install ceph-fuse 2026-02-20T21:04:38.399 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:00 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:38.482 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:38.483 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:38.483 INFO:teuthology.orchestra.run.trial069.stdout: Package Architecture Version Repository Size 2026-02-20T21:04:38.483 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:38.483 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:38.483 INFO:teuthology.orchestra.run.trial069.stdout: ceph-fuse x86_64 2:20.2.0-677.gf7870454.el9 ceph 943 k 2026-02-20T21:04:38.484 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:38.484 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:38.484 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:38.484 INFO:teuthology.orchestra.run.trial069.stdout:Install 1 Package 2026-02-20T21:04:38.484 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:38.484 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 943 k 2026-02-20T21:04:38.484 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 2.7 M 2026-02-20T21:04:38.484 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:38.604 INFO:teuthology.orchestra.run.trial069.stdout:ceph-fuse-20.2.0-677.gf7870454.el9.x86_64.rpm 7.6 MB/s | 943 kB 00:00 2026-02-20T21:04:38.604 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:38.605 INFO:teuthology.orchestra.run.trial069.stdout:Total 7.6 MB/s | 943 kB 00:00 2026-02-20T21:04:38.605 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:38.609 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:38.609 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:38.642 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:38.643 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:38.731 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:38.749 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:38.952 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:39.008 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:39.008 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:39.008 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:39.008 INFO:teuthology.orchestra.run.trial069.stdout: ceph-fuse-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:39.008 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:39.008 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:39.035 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install ceph-volume 2026-02-20T21:04:39.343 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:01 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:39.426 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:39.427 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:39.427 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repository Size 2026-02-20T21:04:39.427 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:39.427 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:39.428 INFO:teuthology.orchestra.run.trial069.stdout: ceph-volume noarch 2:20.2.0-677.gf7870454.el9 ceph-noarch 297 k 2026-02-20T21:04:39.428 INFO:teuthology.orchestra.run.trial069.stdout:Installing dependencies: 2026-02-20T21:04:39.428 INFO:teuthology.orchestra.run.trial069.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-20T21:04:39.428 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:39.428 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:39.428 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:39.428 INFO:teuthology.orchestra.run.trial069.stdout:Install 2 Packages 2026-02-20T21:04:39.428 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:39.428 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 648 k 2026-02-20T21:04:39.428 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 2.2 M 2026-02-20T21:04:39.428 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:39.798 INFO:teuthology.orchestra.run.trial069.stdout:(1/2): cryptsetup-2.8.1-3.el9.x86_64.rpm 3.1 MB/s | 351 kB 00:00 2026-02-20T21:04:40.156 INFO:teuthology.orchestra.run.trial069.stdout:(2/2): ceph-volume-20.2.0-677.gf7870454.el9.noa 635 kB/s | 297 kB 00:00 2026-02-20T21:04:40.157 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:40.157 INFO:teuthology.orchestra.run.trial069.stdout:Total 888 kB/s | 648 kB 00:00 2026-02-20T21:04:40.166 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:40.172 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:40.172 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:40.215 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:40.215 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:40.294 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:40.356 INFO:teuthology.orchestra.run.trial069.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 1/2 2026-02-20T21:04:40.358 INFO:teuthology.orchestra.run.trial069.stdout: Installing : ceph-volume-2:20.2.0-677.gf7870454.el9.noarch 2/2 2026-02-20T21:04:40.373 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: ceph-volume-2:20.2.0-677.gf7870454.el9.noarch 2/2 2026-02-20T21:04:40.373 INFO:teuthology.orchestra.run.trial069.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:40.373 INFO:teuthology.orchestra.run.trial069.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-20T21:04:40.373 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:40.591 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : ceph-volume-2:20.2.0-677.gf7870454.el9.noarch 1/2 2026-02-20T21:04:40.649 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 2/2 2026-02-20T21:04:40.649 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:40.649 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:40.649 INFO:teuthology.orchestra.run.trial069.stdout: ceph-volume-2:20.2.0-677.gf7870454.el9.noarch cryptsetup-2.8.1-3.el9.x86_64 2026-02-20T21:04:40.649 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:40.649 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:40.677 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install librados-devel 2026-02-20T21:04:40.985 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:03 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:41.068 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:41.068 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:41.068 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repo Size 2026-02-20T21:04:41.069 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:41.069 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:41.069 INFO:teuthology.orchestra.run.trial069.stdout: librados-devel x86_64 2:20.2.0-677.gf7870454.el9 ceph 126 k 2026-02-20T21:04:41.069 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:41.069 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:41.069 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:41.069 INFO:teuthology.orchestra.run.trial069.stdout:Install 1 Package 2026-02-20T21:04:41.069 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:41.069 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 126 k 2026-02-20T21:04:41.069 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 449 k 2026-02-20T21:04:41.070 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:41.551 INFO:teuthology.orchestra.run.trial069.stdout:librados-devel-20.2.0-677.gf7870454.el9.x86_64. 261 kB/s | 126 kB 00:00 2026-02-20T21:04:41.551 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:41.551 INFO:teuthology.orchestra.run.trial069.stdout:Total 261 kB/s | 126 kB 00:00 2026-02-20T21:04:41.551 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:41.554 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:41.554 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:41.584 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:41.585 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:41.638 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:41.654 INFO:teuthology.orchestra.run.trial069.stdout: Installing : librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:41.739 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:41.796 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:41.797 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:41.797 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:41.797 INFO:teuthology.orchestra.run.trial069.stdout: librados-devel-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:41.797 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:41.797 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:41.824 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install libcephfs2 2026-02-20T21:04:42.131 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:04 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:42.195 INFO:teuthology.orchestra.run.trial069.stdout:Package libcephfs2-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:42.214 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:42.215 INFO:teuthology.orchestra.run.trial069.stdout:Nothing to do. 2026-02-20T21:04:42.215 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:42.236 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install libcephfs-devel 2026-02-20T21:04:42.542 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:04 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:42.625 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:42.625 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:42.626 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repo Size 2026-02-20T21:04:42.626 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:42.626 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:42.626 INFO:teuthology.orchestra.run.trial069.stdout: libcephfs-devel x86_64 2:20.2.0-677.gf7870454.el9 ceph 34 k 2026-02-20T21:04:42.626 INFO:teuthology.orchestra.run.trial069.stdout:Installing dependencies: 2026-02-20T21:04:42.626 INFO:teuthology.orchestra.run.trial069.stdout: libcephfs-proxy2 x86_64 2:20.2.0-677.gf7870454.el9 ceph 24 k 2026-02-20T21:04:42.626 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:42.626 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:42.626 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:42.626 INFO:teuthology.orchestra.run.trial069.stdout:Install 2 Packages 2026-02-20T21:04:42.627 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:42.627 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 58 k 2026-02-20T21:04:42.627 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 207 k 2026-02-20T21:04:42.627 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:42.689 INFO:teuthology.orchestra.run.trial069.stdout:(1/2): libcephfs-proxy2-20.2.0-677.gf7870454.el 380 kB/s | 24 kB 00:00 2026-02-20T21:04:42.692 INFO:teuthology.orchestra.run.trial069.stdout:(2/2): libcephfs-devel-20.2.0-677.gf7870454.el9 517 kB/s | 34 kB 00:00 2026-02-20T21:04:42.693 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:42.693 INFO:teuthology.orchestra.run.trial069.stdout:Total 866 kB/s | 58 kB 00:00 2026-02-20T21:04:42.693 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:42.696 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:42.696 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:42.728 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:42.728 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:42.778 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:42.779 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 1/2 2026-02-20T21:04:42.793 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 1/2 2026-02-20T21:04:42.809 INFO:teuthology.orchestra.run.trial069.stdout: Installing : libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 2/2 2026-02-20T21:04:42.867 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 2/2 2026-02-20T21:04:42.867 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 1/2 2026-02-20T21:04:42.924 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 2/2 2026-02-20T21:04:42.924 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:42.924 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:42.925 INFO:teuthology.orchestra.run.trial069.stdout: libcephfs-devel-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:42.925 INFO:teuthology.orchestra.run.trial069.stdout: libcephfs-proxy2-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:42.925 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:42.925 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:42.950 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install librados2 2026-02-20T21:04:43.260 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:05 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:43.325 INFO:teuthology.orchestra.run.trial069.stdout:Package librados2-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:43.343 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:43.344 INFO:teuthology.orchestra.run.trial069.stdout:Nothing to do. 2026-02-20T21:04:43.344 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:43.365 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install librbd1 2026-02-20T21:04:43.671 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:05 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:43.736 INFO:teuthology.orchestra.run.trial069.stdout:Package librbd1-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:43.754 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:43.755 INFO:teuthology.orchestra.run.trial069.stdout:Nothing to do. 2026-02-20T21:04:43.755 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:43.776 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install python3-rados 2026-02-20T21:04:44.080 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:06 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:44.146 INFO:teuthology.orchestra.run.trial069.stdout:Package python3-rados-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:44.164 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:44.165 INFO:teuthology.orchestra.run.trial069.stdout:Nothing to do. 2026-02-20T21:04:44.165 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:44.187 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install python3-rgw 2026-02-20T21:04:44.495 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:06 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:44.560 INFO:teuthology.orchestra.run.trial069.stdout:Package python3-rgw-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:44.578 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:44.579 INFO:teuthology.orchestra.run.trial069.stdout:Nothing to do. 2026-02-20T21:04:44.579 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:44.600 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install python3-cephfs 2026-02-20T21:04:44.910 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:06 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:44.975 INFO:teuthology.orchestra.run.trial069.stdout:Package python3-cephfs-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:44.993 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:44.993 INFO:teuthology.orchestra.run.trial069.stdout:Nothing to do. 2026-02-20T21:04:44.994 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:45.014 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install python3-rbd 2026-02-20T21:04:45.323 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:07 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:45.388 INFO:teuthology.orchestra.run.trial069.stdout:Package python3-rbd-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:45.406 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:45.407 INFO:teuthology.orchestra.run.trial069.stdout:Nothing to do. 2026-02-20T21:04:45.407 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:45.429 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install rbd-fuse 2026-02-20T21:04:45.736 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:07 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:45.821 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:45.821 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:45.821 INFO:teuthology.orchestra.run.trial069.stdout: Package Architecture Version Repository Size 2026-02-20T21:04:45.821 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:45.821 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:45.821 INFO:teuthology.orchestra.run.trial069.stdout: rbd-fuse x86_64 2:20.2.0-677.gf7870454.el9 ceph 91 k 2026-02-20T21:04:45.822 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:45.822 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:45.822 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:45.822 INFO:teuthology.orchestra.run.trial069.stdout:Install 1 Package 2026-02-20T21:04:45.822 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:45.822 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 91 k 2026-02-20T21:04:45.822 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 238 k 2026-02-20T21:04:45.822 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:45.906 INFO:teuthology.orchestra.run.trial069.stdout:rbd-fuse-20.2.0-677.gf7870454.el9.x86_64.rpm 1.0 MB/s | 91 kB 00:00 2026-02-20T21:04:45.906 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:45.906 INFO:teuthology.orchestra.run.trial069.stdout:Total 1.0 MB/s | 91 kB 00:00 2026-02-20T21:04:45.906 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:45.911 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:45.911 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:45.942 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:45.943 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:45.993 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:46.010 INFO:teuthology.orchestra.run.trial069.stdout: Installing : rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:46.090 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:46.147 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:46.148 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:46.148 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:46.148 INFO:teuthology.orchestra.run.trial069.stdout: rbd-fuse-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:46.148 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:46.148 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:46.176 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install rbd-mirror 2026-02-20T21:04:46.486 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:08 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:46.570 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:46.570 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:46.570 INFO:teuthology.orchestra.run.trial069.stdout: Package Arch Version Repo Size 2026-02-20T21:04:46.571 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:46.571 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:46.571 INFO:teuthology.orchestra.run.trial069.stdout: rbd-mirror x86_64 2:20.2.0-677.gf7870454.el9 ceph 2.9 M 2026-02-20T21:04:46.571 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:46.571 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:46.571 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:46.571 INFO:teuthology.orchestra.run.trial069.stdout:Install 1 Package 2026-02-20T21:04:46.571 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:46.571 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 2.9 M 2026-02-20T21:04:46.571 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 11 M 2026-02-20T21:04:46.571 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:46.720 INFO:teuthology.orchestra.run.trial069.stdout:rbd-mirror-20.2.0-677.gf7870454.el9.x86_64.rpm 20 MB/s | 2.9 MB 00:00 2026-02-20T21:04:46.720 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:46.720 INFO:teuthology.orchestra.run.trial069.stdout:Total 20 MB/s | 2.9 MB 00:00 2026-02-20T21:04:46.720 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:46.725 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:46.725 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:46.763 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:46.763 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:46.951 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:46.953 INFO:teuthology.orchestra.run.trial069.stdout: Installing : rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:46.976 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:46.976 INFO:teuthology.orchestra.run.trial069.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-20T21:04:46.976 INFO:teuthology.orchestra.run.trial069.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-20T21:04:46.976 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-20T21:04:46.976 INFO:teuthology.orchestra.run.trial069.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-20T21:04:46.976 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:47.299 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:47.300 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:47.300 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:47.300 INFO:teuthology.orchestra.run.trial069.stdout: rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:47.300 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:47.300 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:47.328 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install rbd-nbd 2026-02-20T21:04:47.642 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:09 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:47.725 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:47.725 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:47.725 INFO:teuthology.orchestra.run.trial069.stdout: Package Architecture Version Repository Size 2026-02-20T21:04:47.726 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:47.726 INFO:teuthology.orchestra.run.trial069.stdout:Installing: 2026-02-20T21:04:47.726 INFO:teuthology.orchestra.run.trial069.stdout: rbd-nbd x86_64 2:20.2.0-677.gf7870454.el9 ceph 180 k 2026-02-20T21:04:47.726 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:47.726 INFO:teuthology.orchestra.run.trial069.stdout:Transaction Summary 2026-02-20T21:04:47.726 INFO:teuthology.orchestra.run.trial069.stdout:================================================================================ 2026-02-20T21:04:47.726 INFO:teuthology.orchestra.run.trial069.stdout:Install 1 Package 2026-02-20T21:04:47.726 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:47.726 INFO:teuthology.orchestra.run.trial069.stdout:Total download size: 180 k 2026-02-20T21:04:47.726 INFO:teuthology.orchestra.run.trial069.stdout:Installed size: 498 k 2026-02-20T21:04:47.727 INFO:teuthology.orchestra.run.trial069.stdout:Downloading Packages: 2026-02-20T21:04:47.813 INFO:teuthology.orchestra.run.trial069.stdout:rbd-nbd-20.2.0-677.gf7870454.el9.x86_64.rpm 2.0 MB/s | 180 kB 00:00 2026-02-20T21:04:47.814 INFO:teuthology.orchestra.run.trial069.stdout:-------------------------------------------------------------------------------- 2026-02-20T21:04:47.814 INFO:teuthology.orchestra.run.trial069.stdout:Total 2.0 MB/s | 180 kB 00:00 2026-02-20T21:04:47.814 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction check 2026-02-20T21:04:47.819 INFO:teuthology.orchestra.run.trial069.stdout:Transaction check succeeded. 2026-02-20T21:04:47.819 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction test 2026-02-20T21:04:47.849 INFO:teuthology.orchestra.run.trial069.stdout:Transaction test succeeded. 2026-02-20T21:04:47.849 INFO:teuthology.orchestra.run.trial069.stdout:Running transaction 2026-02-20T21:04:47.904 INFO:teuthology.orchestra.run.trial069.stdout: Preparing : 1/1 2026-02-20T21:04:47.920 INFO:teuthology.orchestra.run.trial069.stdout: Installing : rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:48.001 INFO:teuthology.orchestra.run.trial069.stdout: Running scriptlet: rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:48.059 INFO:teuthology.orchestra.run.trial069.stdout: Verifying : rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 1/1 2026-02-20T21:04:48.059 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:48.059 INFO:teuthology.orchestra.run.trial069.stdout:Installed: 2026-02-20T21:04:48.059 INFO:teuthology.orchestra.run.trial069.stdout: rbd-nbd-2:20.2.0-677.gf7870454.el9.x86_64 2026-02-20T21:04:48.059 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:48.059 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:48.087 DEBUG:teuthology.orchestra.run.trial069:> sudo yum -y install rbd-mirror 2026-02-20T21:04:48.400 INFO:teuthology.orchestra.run.trial069.stdout:Last metadata expiration check: 0:01:10 ago on Fri 20 Feb 2026 09:03:38 PM UTC. 2026-02-20T21:04:48.465 INFO:teuthology.orchestra.run.trial069.stdout:Package rbd-mirror-2:20.2.0-677.gf7870454.el9.x86_64 is already installed. 2026-02-20T21:04:48.483 INFO:teuthology.orchestra.run.trial069.stdout:Dependencies resolved. 2026-02-20T21:04:48.484 INFO:teuthology.orchestra.run.trial069.stdout:Nothing to do. 2026-02-20T21:04:48.484 INFO:teuthology.orchestra.run.trial069.stdout:Complete! 2026-02-20T21:04:48.505 DEBUG:teuthology.parallel:result is None 2026-02-20T21:04:48.505 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:04:48.630 DEBUG:teuthology.orchestra.run.trial069:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-20T21:04:48.650 INFO:teuthology.orchestra.run.trial069.stdout:20.2.0-677.gf7870454.el9 2026-02-20T21:04:48.650 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-677.gf7870454.el9 2026-02-20T21:04:48.650 INFO:teuthology.task.install:The correct ceph version 20.2.0-677.gf7870454 is installed. 2026-02-20T21:04:48.652 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:04:48.799 DEBUG:teuthology.orchestra.run.trial163:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-20T21:04:48.821 INFO:teuthology.orchestra.run.trial163.stdout:20.2.0-677.gf7870454.el9 2026-02-20T21:04:48.822 INFO:teuthology.packaging:The installed version of ceph is 20.2.0-677.gf7870454.el9 2026-02-20T21:04:48.822 INFO:teuthology.task.install:The correct ceph version 20.2.0-677.gf7870454 is installed. 2026-02-20T21:04:48.824 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-20T21:04:48.824 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:48.825 DEBUG:teuthology.orchestra.run.trial069:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-20T21:04:48.850 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:48.850 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-20T21:04:48.886 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-20T21:04:48.886 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:48.887 DEBUG:teuthology.orchestra.run.trial069:> sudo dd of=/usr/bin/daemon-helper 2026-02-20T21:04:48.915 DEBUG:teuthology.orchestra.run.trial069:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-20T21:04:48.981 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:48.981 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/usr/bin/daemon-helper 2026-02-20T21:04:49.007 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-20T21:04:49.070 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-20T21:04:49.071 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:49.071 DEBUG:teuthology.orchestra.run.trial069:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-20T21:04:49.095 DEBUG:teuthology.orchestra.run.trial069:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-20T21:04:49.159 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:49.160 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-20T21:04:49.186 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-20T21:04:49.249 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-20T21:04:49.250 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:49.250 DEBUG:teuthology.orchestra.run.trial069:> sudo dd of=/usr/bin/stdin-killer 2026-02-20T21:04:49.276 DEBUG:teuthology.orchestra.run.trial069:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-20T21:04:49.341 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:49.342 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/usr/bin/stdin-killer 2026-02-20T21:04:49.367 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-20T21:04:49.433 INFO:teuthology.run_tasks:Running task ceph... 2026-02-20T21:04:49.547 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-20T21:04:49.547 DEBUG:teuthology.orchestra.run.trial069:> sudo chmod 777 /var/log/ceph 2026-02-20T21:04:49.549 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod 777 /var/log/ceph 2026-02-20T21:04:49.572 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-20T21:04:49.572 DEBUG:teuthology.orchestra.run.trial069:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-20T21:04:49.614 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-20T21:04:49.637 INFO:tasks.ceph:Creating extra log directories... 2026-02-20T21:04:49.637 DEBUG:teuthology.orchestra.run.trial069:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-20T21:04:49.679 DEBUG:teuthology.orchestra.run.trial163:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-20T21:04:49.710 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-20T21:04:49.710 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-20T21:04:49.710 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424', 'branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-journal-stress-workunit}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '61424', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial', 'no_nested_subset': False, 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-pdonnell-testing-20260219.182737-tentacle'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 7161, 'sha1': 'f7870454149f22574086dd2ff55d050585136531', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f7870454149f22574086dd2ff55d050585136531', 'targets': {'trial069.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEoqTcZenIAF+X4wTlKR02Zcwn/bnCYUvW9fRROi/bwifmBbXs+0+4g9t+pgqRVjFR31ZzHFEE27LW18lMALvwo=', 'trial163.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBI41I1LStMFdfvKEyoHbBOSe5S7OwVwzjasADkZJME4I0YpV+/nK0xUPVNn+RRY2yhe50w+BDaglotviCmH4WSk='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'pool', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-20_16:15:01', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.1540699'} 2026-02-20T21:04:49.711 DEBUG:teuthology.orchestra.run.trial069:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-20T21:04:49.742 DEBUG:teuthology.orchestra.run.trial163:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-20T21:04:49.758 DEBUG:teuthology.orchestra.run.trial069:> sudo install -d -m0777 -- /var/run/ceph 2026-02-20T21:04:49.799 DEBUG:teuthology.orchestra.run.trial163:> sudo install -d -m0777 -- /var/run/ceph 2026-02-20T21:04:49.825 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:49.825 DEBUG:teuthology.orchestra.run.trial069:> dd if=/scratch_devs of=/dev/stdout 2026-02-20T21:04:49.881 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-20T21:04:49.881 DEBUG:teuthology.orchestra.run.trial069:> stat /dev/vg_nvme/lv_1 2026-02-20T21:04:49.938 INFO:teuthology.orchestra.run.trial069.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-20T21:04:49.938 INFO:teuthology.orchestra.run.trial069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:04:49.938 INFO:teuthology.orchestra.run.trial069.stdout:Device: 6h/6d Inode: 908 Links: 1 2026-02-20T21:04:49.939 INFO:teuthology.orchestra.run.trial069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:04:49.939 INFO:teuthology.orchestra.run.trial069.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:04:49.939 INFO:teuthology.orchestra.run.trial069.stdout:Access: 2026-02-20 21:04:47.155011008 +0000 2026-02-20T21:04:49.939 INFO:teuthology.orchestra.run.trial069.stdout:Modify: 2026-02-20 21:03:13.060023035 +0000 2026-02-20T21:04:49.939 INFO:teuthology.orchestra.run.trial069.stdout:Change: 2026-02-20 21:03:13.060023035 +0000 2026-02-20T21:04:49.939 INFO:teuthology.orchestra.run.trial069.stdout: Birth: 2026-02-20 21:03:13.060023035 +0000 2026-02-20T21:04:49.940 DEBUG:teuthology.orchestra.run.trial069:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-20T21:04:50.001 INFO:teuthology.orchestra.run.trial069.stderr:1+0 records in 2026-02-20T21:04:50.002 INFO:teuthology.orchestra.run.trial069.stderr:1+0 records out 2026-02-20T21:04:50.002 INFO:teuthology.orchestra.run.trial069.stderr:512 bytes copied, 0.000148789 s, 3.4 MB/s 2026-02-20T21:04:50.003 DEBUG:teuthology.orchestra.run.trial069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-20T21:04:50.062 DEBUG:teuthology.orchestra.run.trial069:> stat /dev/vg_nvme/lv_2 2026-02-20T21:04:50.118 INFO:teuthology.orchestra.run.trial069.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-20T21:04:50.118 INFO:teuthology.orchestra.run.trial069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:04:50.118 INFO:teuthology.orchestra.run.trial069.stdout:Device: 6h/6d Inode: 937 Links: 1 2026-02-20T21:04:50.118 INFO:teuthology.orchestra.run.trial069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:04:50.118 INFO:teuthology.orchestra.run.trial069.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:04:50.118 INFO:teuthology.orchestra.run.trial069.stdout:Access: 2026-02-20 21:04:47.155011008 +0000 2026-02-20T21:04:50.118 INFO:teuthology.orchestra.run.trial069.stdout:Modify: 2026-02-20 21:03:13.299021220 +0000 2026-02-20T21:04:50.118 INFO:teuthology.orchestra.run.trial069.stdout:Change: 2026-02-20 21:03:13.299021220 +0000 2026-02-20T21:04:50.118 INFO:teuthology.orchestra.run.trial069.stdout: Birth: 2026-02-20 21:03:13.299021220 +0000 2026-02-20T21:04:50.118 DEBUG:teuthology.orchestra.run.trial069:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-20T21:04:50.180 INFO:teuthology.orchestra.run.trial069.stderr:1+0 records in 2026-02-20T21:04:50.180 INFO:teuthology.orchestra.run.trial069.stderr:1+0 records out 2026-02-20T21:04:50.180 INFO:teuthology.orchestra.run.trial069.stderr:512 bytes copied, 0.00014962 s, 3.4 MB/s 2026-02-20T21:04:50.181 DEBUG:teuthology.orchestra.run.trial069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-20T21:04:50.238 DEBUG:teuthology.orchestra.run.trial069:> stat /dev/vg_nvme/lv_3 2026-02-20T21:04:50.293 INFO:teuthology.orchestra.run.trial069.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-20T21:04:50.294 INFO:teuthology.orchestra.run.trial069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:04:50.294 INFO:teuthology.orchestra.run.trial069.stdout:Device: 6h/6d Inode: 966 Links: 1 2026-02-20T21:04:50.294 INFO:teuthology.orchestra.run.trial069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:04:50.294 INFO:teuthology.orchestra.run.trial069.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:04:50.294 INFO:teuthology.orchestra.run.trial069.stdout:Access: 2026-02-20 21:04:47.155011008 +0000 2026-02-20T21:04:50.294 INFO:teuthology.orchestra.run.trial069.stdout:Modify: 2026-02-20 21:03:13.542019373 +0000 2026-02-20T21:04:50.294 INFO:teuthology.orchestra.run.trial069.stdout:Change: 2026-02-20 21:03:13.542019373 +0000 2026-02-20T21:04:50.294 INFO:teuthology.orchestra.run.trial069.stdout: Birth: 2026-02-20 21:03:13.542019373 +0000 2026-02-20T21:04:50.294 DEBUG:teuthology.orchestra.run.trial069:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-20T21:04:50.356 INFO:teuthology.orchestra.run.trial069.stderr:1+0 records in 2026-02-20T21:04:50.356 INFO:teuthology.orchestra.run.trial069.stderr:1+0 records out 2026-02-20T21:04:50.356 INFO:teuthology.orchestra.run.trial069.stderr:512 bytes copied, 0.000152074 s, 3.4 MB/s 2026-02-20T21:04:50.357 DEBUG:teuthology.orchestra.run.trial069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-20T21:04:50.415 DEBUG:teuthology.orchestra.run.trial069:> stat /dev/vg_nvme/lv_4 2026-02-20T21:04:50.471 INFO:teuthology.orchestra.run.trial069.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-20T21:04:50.471 INFO:teuthology.orchestra.run.trial069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:04:50.471 INFO:teuthology.orchestra.run.trial069.stdout:Device: 6h/6d Inode: 992 Links: 1 2026-02-20T21:04:50.471 INFO:teuthology.orchestra.run.trial069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:04:50.471 INFO:teuthology.orchestra.run.trial069.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:04:50.471 INFO:teuthology.orchestra.run.trial069.stdout:Access: 2026-02-20 21:04:47.155011008 +0000 2026-02-20T21:04:50.471 INFO:teuthology.orchestra.run.trial069.stdout:Modify: 2026-02-20 21:03:13.797017433 +0000 2026-02-20T21:04:50.471 INFO:teuthology.orchestra.run.trial069.stdout:Change: 2026-02-20 21:03:13.797017433 +0000 2026-02-20T21:04:50.471 INFO:teuthology.orchestra.run.trial069.stdout: Birth: 2026-02-20 21:03:13.797017433 +0000 2026-02-20T21:04:50.472 DEBUG:teuthology.orchestra.run.trial069:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-20T21:04:50.533 INFO:teuthology.orchestra.run.trial069.stderr:1+0 records in 2026-02-20T21:04:50.533 INFO:teuthology.orchestra.run.trial069.stderr:1+0 records out 2026-02-20T21:04:50.533 INFO:teuthology.orchestra.run.trial069.stderr:512 bytes copied, 0.000138199 s, 3.7 MB/s 2026-02-20T21:04:50.534 DEBUG:teuthology.orchestra.run.trial069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-20T21:04:50.591 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:04:50.591 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial069.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-20T21:04:50.591 INFO:tasks.ceph:Generating config... 2026-02-20T21:04:50.592 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-20T21:04:50.592 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-20T21:04:50.593 INFO:tasks.ceph:[client] log to stderr = False 2026-02-20T21:04:50.593 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-20T21:04:50.593 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2026-02-20T21:04:50.593 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:04:50.593 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:04:50.593 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:04:50.593 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:04:50.593 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:04:50.593 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:04:50.593 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:04:50.593 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:04:50.593 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-20T21:04:50.593 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-20T21:04:50.593 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-20T21:04:50.593 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-20T21:04:50.593 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-20T21:04:50.593 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-20T21:04:50.593 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-20T21:04:50.593 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-20T21:04:50.593 INFO:tasks.ceph:[osd] bdev async discard = True 2026-02-20T21:04:50.593 INFO:tasks.ceph:[osd] bdev enable discard = True 2026-02-20T21:04:50.593 INFO:tasks.ceph:[osd] bluestore allocator = stupid 2026-02-20T21:04:50.593 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-20T21:04:50.594 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-20T21:04:50.594 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-20T21:04:50.594 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-20T21:04:50.594 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-20T21:04:50.594 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-20T21:04:50.594 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-20T21:04:50.594 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-20T21:04:50.594 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-20T21:04:50.594 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-20T21:04:50.594 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-20T21:04:50.594 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-20T21:04:50.594 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-20T21:04:50.594 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-20T21:04:50.665 INFO:teuthology.orchestra.run.trial069.stdout:creating /etc/ceph/cluster1.keyring 2026-02-20T21:04:50.668 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-20T21:04:50.745 DEBUG:teuthology.orchestra.run.trial069:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-20T21:04:50.808 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.69')] 2026-02-20T21:04:50.808 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.69', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-20T21:04:50.808 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:50.808 DEBUG:teuthology.orchestra.run.trial069:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-20T21:04:50.865 DEBUG:teuthology.orchestra.run.trial069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.69 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:04:50.937 INFO:teuthology.orchestra.run.trial069.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:04:50.937 INFO:teuthology.orchestra.run.trial069.stdout:monmaptool: generated fsid 4aabec94-a67f-4d24-9557-469e7f74026e 2026-02-20T21:04:50.937 INFO:teuthology.orchestra.run.trial069.stdout:setting min_mon_release = tentacle 2026-02-20T21:04:50.937 INFO:teuthology.orchestra.run.trial069.stdout:epoch 0 2026-02-20T21:04:50.937 INFO:teuthology.orchestra.run.trial069.stdout:fsid 4aabec94-a67f-4d24-9557-469e7f74026e 2026-02-20T21:04:50.937 INFO:teuthology.orchestra.run.trial069.stdout:last_changed 2026-02-20T21:04:50.967908+0000 2026-02-20T21:04:50.937 INFO:teuthology.orchestra.run.trial069.stdout:created 2026-02-20T21:04:50.967908+0000 2026-02-20T21:04:50.937 INFO:teuthology.orchestra.run.trial069.stdout:min_mon_release 20 (tentacle) 2026-02-20T21:04:50.938 INFO:teuthology.orchestra.run.trial069.stdout:election_strategy: 1 2026-02-20T21:04:50.938 INFO:teuthology.orchestra.run.trial069.stdout:0: [v2:10.20.193.69:3300/0,v1:10.20.193.69:6789/0] mon.a 2026-02-20T21:04:50.938 INFO:teuthology.orchestra.run.trial069.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-20T21:04:50.939 DEBUG:teuthology.orchestra.run.trial069:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-20T21:04:50.996 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 4aabec94-a67f-4d24-9557-469e7f74026e... 2026-02-20T21:04:50.996 DEBUG:teuthology.orchestra.run.trial069:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-20T21:04:51.038 DEBUG:teuthology.orchestra.run.trial163:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout:[global] 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: chdir = "" 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: auth supported = cephx 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: filestore xattr use omap = true 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: mon clock drift allowed = 1.000 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: osd crush chooseleaf type = 0 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: auth debug = true 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: ms die on old message = true 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: ms die on bug = true 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: mon pg warn max object skew = 0 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: # disable pg_autoscaler by default for new pools 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: osd pool default size = 2 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: mon osd allow primary affinity = true 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: mon osd allow pg remap = true 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: mon warn on legacy crush tunables = false 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: mon warn on crush straw calc version zero = false 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: mon warn on no sortbitwise = false 2026-02-20T21:04:51.077 INFO:teuthology.orchestra.run.trial069.stdout: mon warn on osd down out interval zero = false 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: mon warn on too few osds = false 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: mon_allow_pool_size_one = true 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd default data pool replay window = 5 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: mon allow pool delete = true 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: mon cluster log file level = debug 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: debug asserts on shutdown = true 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: mon health detail to clog = false 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: mon host = 10.20.193.69 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: mon client directed command retry = 5 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: mon warn on pool no app = False 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: ms inject socket failures = 5000 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: fsid = 4aabec94-a67f-4d24-9557-469e7f74026e 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout:[osd] 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd journal size = 100 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd scrub load threshold = 5.0 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd scrub max interval = 600 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd mclock profile = high_recovery_ops 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd mclock skip benchmark = true 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd recover clone overlap = true 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd recovery max chunk = 1048576 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd debug shutdown = true 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd debug op order = true 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd debug verify stray on activate = true 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd debug trim objects = true 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd open classes on start = true 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd debug pg log writeout = true 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd deep scrub update digest min age = 30 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: osd map max advance = 10 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: journal zero on create = true 2026-02-20T21:04:51.078 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: filestore ondisk finisher threads = 3 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: filestore apply finisher threads = 3 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: bdev debug aio = true 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: osd debug misdirected ops = true 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: bdev async discard = True 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: bdev enable discard = True 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: bluestore allocator = stupid 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: bluestore block size = 96636764160 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: bluestore fsck on mount = True 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug bluefs = 1/20 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug bluestore = 1/20 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug ms = 1 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug osd = 20 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug rocksdb = 4/10 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: mon osd backfillfull_ratio = 0.85 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: mon osd full ratio = 0.9 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: mon osd nearfull ratio = 0.8 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: osd failsafe full ratio = 0.95 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: osd objectstore = bluestore 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout:[mgr] 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug ms = 1 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug mgr = 20 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug mon = 20 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug auth = 20 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: mgr/telemetry/nag = false 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout:[mon] 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug ms = 1 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug mon = 20 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug paxos = 20 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: debug auth = 20 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: mon data avail warn = 5 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: mon mgr mkfs grace = 240 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: mon osd reporter subtree level = osd 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: mon osd prime pg temp = true 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: auth mon ticket ttl = 660 # 11m 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: auth service ticket ttl = 240 # 4m 2026-02-20T21:04:51.079 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: # don't complain about insecure global_id in the test suite 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: # 1m isn't quite enough 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: mon_down_mkfs_grace = 2m 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: mon_warn_on_filestore_osds = false 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout:[client] 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: rgw cache enabled = true 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: rgw enable ops log = true 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: rgw enable usage log = true 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: debug rbd = 20 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: debug rbd_mirror = 15 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: log to stderr = False 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: rbd default features = 125 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: rbd mirror image policy type = simple 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout:[mon.a] 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.0] 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.1] 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.2] 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.3] 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial069.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial163.stdout:[global] 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial163.stdout: chdir = "" 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-20T21:04:51.080 INFO:teuthology.orchestra.run.trial163.stdout: auth supported = cephx 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: filestore xattr use omap = true 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon clock drift allowed = 1.000 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: osd crush chooseleaf type = 0 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: auth debug = true 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: ms die on old message = true 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: ms die on bug = true 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon pg warn max object skew = 0 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: # disable pg_autoscaler by default for new pools 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: osd pool default size = 2 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon osd allow primary affinity = true 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon osd allow pg remap = true 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on legacy crush tunables = false 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on crush straw calc version zero = false 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on no sortbitwise = false 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on osd down out interval zero = false 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on too few osds = false 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon_allow_pool_size_one = true 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: osd default data pool replay window = 5 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon allow pool delete = true 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon cluster log file level = debug 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: debug asserts on shutdown = true 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon health detail to clog = false 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon host = 10.20.193.69 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon client directed command retry = 5 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on pool no app = False 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: ms inject socket failures = 5000 2026-02-20T21:04:51.081 INFO:teuthology.orchestra.run.trial163.stdout: fsid = 4aabec94-a67f-4d24-9557-469e7f74026e 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout:[osd] 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd journal size = 100 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd scrub load threshold = 5.0 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd scrub max interval = 600 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd mclock profile = high_recovery_ops 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd mclock skip benchmark = true 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd recover clone overlap = true 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd recovery max chunk = 1048576 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd debug shutdown = true 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd debug op order = true 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd debug verify stray on activate = true 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd debug trim objects = true 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd open classes on start = true 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd debug pg log writeout = true 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd deep scrub update digest min age = 30 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd map max advance = 10 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: journal zero on create = true 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: filestore ondisk finisher threads = 3 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: filestore apply finisher threads = 3 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: bdev debug aio = true 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd debug misdirected ops = true 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: bdev async discard = True 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: bdev enable discard = True 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: bluestore allocator = stupid 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: bluestore block size = 96636764160 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: bluestore fsck on mount = True 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: debug bluefs = 1/20 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: debug bluestore = 1/20 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: debug ms = 1 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: debug osd = 20 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: debug rocksdb = 4/10 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: mon osd backfillfull_ratio = 0.85 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: mon osd full ratio = 0.9 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: mon osd nearfull ratio = 0.8 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd failsafe full ratio = 0.95 2026-02-20T21:04:51.082 INFO:teuthology.orchestra.run.trial163.stdout: osd objectstore = bluestore 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout:[mgr] 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: debug ms = 1 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: debug mgr = 20 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: debug mon = 20 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: debug auth = 20 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mgr/telemetry/nag = false 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout:[mon] 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: debug ms = 1 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: debug mon = 20 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: debug paxos = 20 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: debug auth = 20 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mon data avail warn = 5 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mon mgr mkfs grace = 240 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mon osd reporter subtree level = osd 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mon osd prime pg temp = true 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: auth mon ticket ttl = 660 # 11m 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: auth service ticket ttl = 240 # 4m 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: # don't complain about insecure global_id in the test suite 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: # 1m isn't quite enough 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mon_down_mkfs_grace = 2m 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_filestore_osds = false 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout:[client] 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: rgw cache enabled = true 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: rgw enable ops log = true 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: rgw enable usage log = true 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: debug rbd = 20 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: debug rbd_mirror = 15 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: log to stderr = False 2026-02-20T21:04:51.083 INFO:teuthology.orchestra.run.trial163.stdout: rbd default features = 125 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout: rbd mirror image policy type = simple 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout:[mon.a] 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.0] 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.1] 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.2] 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.3] 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:04:51.084 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:04:51.087 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-20T21:04:51.087 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-20T21:04:51.168 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-20T21:04:51.168 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:51.168 DEBUG:teuthology.orchestra.run.trial069:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-20T21:04:51.225 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:51.225 DEBUG:teuthology.orchestra.run.trial069:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-20T21:04:51.281 INFO:tasks.ceph:Sending monmap to node ubuntu@trial069.front.sepia.ceph.com 2026-02-20T21:04:51.282 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:51.282 DEBUG:teuthology.orchestra.run.trial069:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-20T21:04:51.282 DEBUG:teuthology.orchestra.run.trial069:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-20T21:04:51.355 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:51.355 DEBUG:teuthology.orchestra.run.trial069:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:04:51.412 INFO:tasks.ceph:Sending monmap to node ubuntu@trial163.front.sepia.ceph.com 2026-02-20T21:04:51.412 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:51.412 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-20T21:04:51.413 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-20T21:04:51.448 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:51.448 DEBUG:teuthology.orchestra.run.trial163:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:04:51.506 INFO:tasks.ceph:Setting up mon nodes... 2026-02-20T21:04:51.507 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-20T21:04:51.507 DEBUG:teuthology.orchestra.run.trial069:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-20T21:04:51.553 INFO:teuthology.orchestra.run.trial069.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-20T21:04:51.556 INFO:tasks.ceph:Setting up mds nodes... 2026-02-20T21:04:51.556 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-20T21:04:51.556 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-20T21:04:51.591 INFO:teuthology.orchestra.run.trial069.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-20T21:04:51.603 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-20T21:04:51.640 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-20T21:04:51.651 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-20T21:04:51.726 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-20T21:04:51.738 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-20T21:04:51.814 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-20T21:04:51.825 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-20T21:04:51.900 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-20T21:04:51.911 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-20T21:04:51.984 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-20T21:04:51.995 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-20T21:04:52.068 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-20T21:04:52.078 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-20T21:04:52.151 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-20T21:04:52.162 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-20T21:04:52.236 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-20T21:04:52.247 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-20T21:04:52.247 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial069.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-20T21:04:52.247 DEBUG:teuthology.orchestra.run.trial069:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-20T21:04:52.274 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:04:52.274 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-20T21:04:52.274 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial069.front.sepia.ceph.com 2026-02-20T21:04:52.274 DEBUG:teuthology.orchestra.run.trial069:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-20T21:04:52.339 INFO:teuthology.orchestra.run.trial069.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:04:52.340 INFO:teuthology.orchestra.run.trial069.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:04:52.340 INFO:teuthology.orchestra.run.trial069.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:04:52.340 INFO:teuthology.orchestra.run.trial069.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-20T21:04:52.340 INFO:teuthology.orchestra.run.trial069.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:04:52.340 INFO:teuthology.orchestra.run.trial069.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:04:52.340 INFO:teuthology.orchestra.run.trial069.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:04:52.340 INFO:teuthology.orchestra.run.trial069.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:04:52.340 INFO:teuthology.orchestra.run.trial069.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:04:52.341 INFO:teuthology.orchestra.run.trial069.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:04:52.964 INFO:teuthology.orchestra.run.trial069.stdout:Discarding blocks...Done. 2026-02-20T21:04:52.966 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial069.front.sepia.ceph.com -o noatime 2026-02-20T21:04:52.966 DEBUG:teuthology.orchestra.run.trial069:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-20T21:04:53.034 DEBUG:teuthology.orchestra.run.trial069:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-20T21:04:53.102 DEBUG:teuthology.orchestra.run.trial069:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-20T21:04:53.166 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:04:53.167 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-20T21:04:53.167 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial069.front.sepia.ceph.com 2026-02-20T21:04:53.167 DEBUG:teuthology.orchestra.run.trial069:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-20T21:04:53.231 INFO:teuthology.orchestra.run.trial069.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:04:53.231 INFO:teuthology.orchestra.run.trial069.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:04:53.231 INFO:teuthology.orchestra.run.trial069.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:04:53.231 INFO:teuthology.orchestra.run.trial069.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-20T21:04:53.231 INFO:teuthology.orchestra.run.trial069.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:04:53.231 INFO:teuthology.orchestra.run.trial069.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:04:53.231 INFO:teuthology.orchestra.run.trial069.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:04:53.231 INFO:teuthology.orchestra.run.trial069.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:04:53.231 INFO:teuthology.orchestra.run.trial069.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:04:53.231 INFO:teuthology.orchestra.run.trial069.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:04:53.792 INFO:teuthology.orchestra.run.trial069.stdout:Discarding blocks...Done. 2026-02-20T21:04:53.794 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial069.front.sepia.ceph.com -o noatime 2026-02-20T21:04:53.794 DEBUG:teuthology.orchestra.run.trial069:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-20T21:04:53.861 DEBUG:teuthology.orchestra.run.trial069:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-20T21:04:53.928 DEBUG:teuthology.orchestra.run.trial069:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-20T21:04:53.990 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:04:53.990 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-20T21:04:53.990 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial069.front.sepia.ceph.com 2026-02-20T21:04:53.990 DEBUG:teuthology.orchestra.run.trial069:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-20T21:04:54.055 INFO:teuthology.orchestra.run.trial069.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:04:54.055 INFO:teuthology.orchestra.run.trial069.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:04:54.055 INFO:teuthology.orchestra.run.trial069.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:04:54.055 INFO:teuthology.orchestra.run.trial069.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-20T21:04:54.055 INFO:teuthology.orchestra.run.trial069.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:04:54.055 INFO:teuthology.orchestra.run.trial069.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:04:54.055 INFO:teuthology.orchestra.run.trial069.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:04:54.055 INFO:teuthology.orchestra.run.trial069.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:04:54.055 INFO:teuthology.orchestra.run.trial069.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:04:54.056 INFO:teuthology.orchestra.run.trial069.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:04:54.661 INFO:teuthology.orchestra.run.trial069.stdout:Discarding blocks...Done. 2026-02-20T21:04:54.663 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial069.front.sepia.ceph.com -o noatime 2026-02-20T21:04:54.663 DEBUG:teuthology.orchestra.run.trial069:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-20T21:04:54.732 DEBUG:teuthology.orchestra.run.trial069:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-20T21:04:54.799 DEBUG:teuthology.orchestra.run.trial069:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:04:54.877 INFO:teuthology.orchestra.run.trial069.stderr:2026-02-20T21:04:54.907+0000 7fd6c99e0900 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-20T21:04:54.878 INFO:teuthology.orchestra.run.trial069.stderr:2026-02-20T21:04:54.907+0000 7fd6c99e0900 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-20T21:04:54.878 INFO:teuthology.orchestra.run.trial069.stderr:2026-02-20T21:04:54.908+0000 7fd6c99e0900 -1 bdev(0x55b8e04f5800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-20T21:04:54.878 INFO:teuthology.orchestra.run.trial069.stderr:2026-02-20T21:04:54.908+0000 7fd6c99e0900 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-20T21:04:55.327 DEBUG:teuthology.orchestra.run.trial069:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-20T21:04:55.393 DEBUG:teuthology.orchestra.run.trial069:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:04:55.470 INFO:teuthology.orchestra.run.trial069.stderr:2026-02-20T21:04:55.500+0000 7fbcd1979900 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-20T21:04:55.470 INFO:teuthology.orchestra.run.trial069.stderr:2026-02-20T21:04:55.500+0000 7fbcd1979900 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-20T21:04:55.470 INFO:teuthology.orchestra.run.trial069.stderr:2026-02-20T21:04:55.500+0000 7fbcd1979900 -1 bdev(0x560624c3f800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-20T21:04:55.470 INFO:teuthology.orchestra.run.trial069.stderr:2026-02-20T21:04:55.500+0000 7fbcd1979900 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-20T21:04:55.937 DEBUG:teuthology.orchestra.run.trial069:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-20T21:04:56.001 DEBUG:teuthology.orchestra.run.trial069:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:04:56.079 INFO:teuthology.orchestra.run.trial069.stderr:2026-02-20T21:04:56.109+0000 7f474a7d5900 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-20T21:04:56.080 INFO:teuthology.orchestra.run.trial069.stderr:2026-02-20T21:04:56.109+0000 7f474a7d5900 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-20T21:04:56.080 INFO:teuthology.orchestra.run.trial069.stderr:2026-02-20T21:04:56.109+0000 7f474a7d5900 -1 bdev(0x561946691800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-20T21:04:56.080 INFO:teuthology.orchestra.run.trial069.stderr:2026-02-20T21:04:56.109+0000 7f474a7d5900 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-20T21:04:56.551 DEBUG:teuthology.orchestra.run.trial069:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-20T21:04:56.619 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-20T21:04:56.619 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:56.619 DEBUG:teuthology.orchestra.run.trial069:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-20T21:04:56.682 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:56.682 DEBUG:teuthology.orchestra.run.trial069:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-20T21:04:56.748 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:56.749 DEBUG:teuthology.orchestra.run.trial069:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-20T21:04:56.811 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:56.812 DEBUG:teuthology.orchestra.run.trial069:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-20T21:04:56.875 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:56.875 DEBUG:teuthology.orchestra.run.trial069:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-20T21:04:56.932 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:56.932 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-20T21:04:56.950 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:56.950 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-20T21:04:57.006 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:57.006 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-20T21:04:57.062 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:57.062 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-20T21:04:57.121 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:57.121 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-20T21:04:57.179 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:57.179 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-20T21:04:57.238 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:57.238 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-20T21:04:57.295 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:04:57.295 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-20T21:04:57.352 INFO:tasks.ceph:Adding keys to all mons... 2026-02-20T21:04:57.352 DEBUG:teuthology.orchestra.run.trial069:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-20T21:04:57.377 INFO:teuthology.orchestra.run.trial069.stdout:[mgr.x] 2026-02-20T21:04:57.378 INFO:teuthology.orchestra.run.trial069.stdout: key = AQBzzJhpDN7KIhAAY7nM/94DLM4ZDrez+hqirg== 2026-02-20T21:04:57.378 INFO:teuthology.orchestra.run.trial069.stdout:[osd.0] 2026-02-20T21:04:57.378 INFO:teuthology.orchestra.run.trial069.stdout: key = AQB2zJhp1CwgNhAAO3bONIHXju6IjZNmAA4wgg== 2026-02-20T21:04:57.378 INFO:teuthology.orchestra.run.trial069.stdout:[osd.1] 2026-02-20T21:04:57.378 INFO:teuthology.orchestra.run.trial069.stdout: key = AQB3zJhpt3XZHRAASFxd7PTRC9aoEt6/B+8hbQ== 2026-02-20T21:04:57.378 INFO:teuthology.orchestra.run.trial069.stdout:[osd.2] 2026-02-20T21:04:57.378 INFO:teuthology.orchestra.run.trial069.stdout: key = AQB4zJhpkdeHBhAAko3iEdjGR3LS+R3aC/vwAw== 2026-02-20T21:04:57.378 INFO:teuthology.orchestra.run.trial069.stdout:[client.0] 2026-02-20T21:04:57.378 INFO:teuthology.orchestra.run.trial069.stdout: key = AQBzzJhpO74JJRAAobt9Drqhsdr8mxOJ4jewWw== 2026-02-20T21:04:57.379 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror] 2026-02-20T21:04:57.379 INFO:teuthology.orchestra.run.trial069.stdout: key = AQBzzJhp5QUjJhAASSPijdDFaZGDomWwbZMQ8Q== 2026-02-20T21:04:57.379 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.0] 2026-02-20T21:04:57.379 INFO:teuthology.orchestra.run.trial069.stdout: key = AQBzzJhpQMNJKxAApZdPirQ1TNbz3+lAV3EHGg== 2026-02-20T21:04:57.379 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.1] 2026-02-20T21:04:57.379 INFO:teuthology.orchestra.run.trial069.stdout: key = AQBzzJhpYW+HMBAASMQhrR1r7dMCWM4Ihx3yRg== 2026-02-20T21:04:57.379 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.2] 2026-02-20T21:04:57.379 INFO:teuthology.orchestra.run.trial069.stdout: key = AQBzzJhpi/qqNRAA50IdPKbE0dVm3KNcCWMBQA== 2026-02-20T21:04:57.379 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.3] 2026-02-20T21:04:57.379 INFO:teuthology.orchestra.run.trial069.stdout: key = AQBzzJhpBECuOhAA570yrT9FfMr6to2tYoZpaA== 2026-02-20T21:04:57.379 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.4] 2026-02-20T21:04:57.380 INFO:teuthology.orchestra.run.trial069.stdout: key = AQB0zJhpJbsRBBAARKlaqBBHabXzDaPmuhPTFw== 2026-02-20T21:04:57.380 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.5] 2026-02-20T21:04:57.380 INFO:teuthology.orchestra.run.trial069.stdout: key = AQB0zJhpxDILCRAAlMAyw5FxYboL0T2G8dBeVA== 2026-02-20T21:04:57.380 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.6] 2026-02-20T21:04:57.380 INFO:teuthology.orchestra.run.trial069.stdout: key = AQB0zJhpq0kODhAARhnnqSTlW1ThdzeEB/ELrA== 2026-02-20T21:04:57.381 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-20T21:04:57.463 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:04:57.544 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:04:57.623 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:04:57.705 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:04:57.786 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:04:57.867 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:04:57.949 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:04:58.029 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:04:58.113 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:04:58.196 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:04:58.279 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:04:58.361 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:04:58.445 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-20T21:04:58.445 DEBUG:teuthology.orchestra.run.trial069:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-20T21:04:58.510 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-20T21:04:58.596 DEBUG:teuthology.orchestra.run.trial069:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-20T21:04:58.621 DEBUG:teuthology.orchestra.run.trial069:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-20T21:04:58.677 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-20T21:04:58.677 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-20T21:04:58.678 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-20T21:04:58.719 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-20T21:04:58.719 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-20T21:04:58.719 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-20T21:04:58.719 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-20T21:04:58.720 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-20T21:04:58.720 DEBUG:tasks.ceph:set 0 configs 2026-02-20T21:04:58.720 DEBUG:teuthology.orchestra.run.trial069:> sudo ceph --cluster cluster1 config dump 2026-02-20T21:04:58.820 INFO:teuthology.orchestra.run.trial069.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-02-20T21:04:58.825 INFO:tasks.ceph:Setting crush tunables to default 2026-02-20T21:04:58.825 DEBUG:teuthology.orchestra.run.trial069:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-20T21:04:58.926 INFO:teuthology.orchestra.run.trial069.stderr:adjusted tunables profile to default 2026-02-20T21:04:58.933 INFO:tasks.ceph:check_enable_crimson: False 2026-02-20T21:04:58.933 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-20T21:04:58.933 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:58.933 DEBUG:teuthology.orchestra.run.trial069:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-20T21:04:58.957 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:58.957 DEBUG:teuthology.orchestra.run.trial069:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-20T21:04:59.022 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:04:59.022 DEBUG:teuthology.orchestra.run.trial069:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-20T21:04:59.086 DEBUG:teuthology.orchestra.run.trial069:> sudo ceph --cluster cluster1 osd new 567f2427-666c-49f4-aff5-54e91a2350f8 0 2026-02-20T21:04:59.229 INFO:teuthology.orchestra.run.trial069.stdout:0 2026-02-20T21:04:59.236 DEBUG:teuthology.orchestra.run.trial069:> sudo ceph --cluster cluster1 osd new c06ca60d-cc95-4882-95d9-c4751ba99875 1 2026-02-20T21:04:59.341 INFO:teuthology.orchestra.run.trial069.stdout:1 2026-02-20T21:04:59.350 DEBUG:teuthology.orchestra.run.trial069:> sudo ceph --cluster cluster1 osd new 6805f378-c47b-4987-8d01-3c839ec7d272 2 2026-02-20T21:04:59.454 INFO:teuthology.orchestra.run.trial069.stdout:2 2026-02-20T21:04:59.463 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-20T21:04:59.463 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-20T21:04:59.466 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-20T21:04:59.466 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-20T21:04:59.466 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-20T21:04:59.469 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-20T21:04:59.469 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-20T21:04:59.469 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-20T21:04:59.471 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-20T21:04:59.472 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:04:59.576 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:04:59.576 INFO:teuthology.orchestra.run.trial069.stdout:{"epoch":5,"fsid":"4aabec94-a67f-4d24-9557-469e7f74026e","created":"2026-02-20T21:04:58.813036+0000","modified":"2026-02-20T21:04:59.484260+0000","last_up_change":"0.000000","last_in_change":"2026-02-20T21:04:59.484260+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"567f2427-666c-49f4-aff5-54e91a2350f8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"c06ca60d-cc95-4882-95d9-c4751ba99875","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"6805f378-c47b-4987-8d01-3c839ec7d272","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:04:59.585 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-20T21:04:59.585 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-20T21:04:59.761 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:04:59.790+0000 7fd68cfcd900 -1 Falling back to public interface 2026-02-20T21:04:59.768 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:04:59.798+0000 7f414f995900 -1 Falling back to public interface 2026-02-20T21:04:59.776 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:04:59.806+0000 7fc9e24d2900 -1 Falling back to public interface 2026-02-20T21:04:59.810 INFO:tasks.ceph.cluster1.mgr.x.trial069.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-20T21:04:59.810 INFO:tasks.ceph.cluster1.mgr.x.trial069.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-20T21:04:59.810 INFO:tasks.ceph.cluster1.mgr.x.trial069.stderr: from numpy import show_config as show_numpy_config 2026-02-20T21:04:59.888 DEBUG:teuthology.orchestra.run.trial069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:04:59.981 INFO:teuthology.misc.health.trial069.stdout: 2026-02-20T21:04:59.981 INFO:teuthology.misc.health.trial069.stdout:{"epoch":5,"fsid":"4aabec94-a67f-4d24-9557-469e7f74026e","created":"2026-02-20T21:04:58.813036+0000","modified":"2026-02-20T21:04:59.484260+0000","last_up_change":"0.000000","last_in_change":"2026-02-20T21:04:59.484260+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"567f2427-666c-49f4-aff5-54e91a2350f8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"c06ca60d-cc95-4882-95d9-c4751ba99875","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"6805f378-c47b-4987-8d01-3c839ec7d272","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:04:59.986 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-20T21:05:00.371 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:05:00.401+0000 7fd68cfcd900 -1 osd.0 0 log_to_monitors true 2026-02-20T21:05:00.373 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:05:00.403+0000 7fc9e24d2900 -1 osd.2 0 log_to_monitors true 2026-02-20T21:05:00.378 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:05:00.407+0000 7f414f995900 -1 osd.1 0 log_to_monitors true 2026-02-20T21:05:01.793 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:05:01.822+0000 7f414b100640 -1 osd.1 0 waiting for initial osdmap 2026-02-20T21:05:01.793 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:05:01.822+0000 7fc9dec74640 -1 osd.2 0 waiting for initial osdmap 2026-02-20T21:05:01.793 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:05:01.823+0000 7fd688738640 -1 osd.0 0 waiting for initial osdmap 2026-02-20T21:05:01.802 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:05:01.832+0000 7f4142f10640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:05:01.803 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:05:01.832+0000 7fc9d5a4f640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:05:01.805 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:05:01.835+0000 7fd680548640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:05:02.016 INFO:tasks.ceph.cluster1.mgr.x.trial069.stderr:2026-02-20T21:05:02.046+0000 7f16f6685640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-20T21:05:06.289 DEBUG:teuthology.orchestra.run.trial069:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:05:06.460 INFO:teuthology.misc.health.trial069.stdout: 2026-02-20T21:05:06.461 INFO:teuthology.misc.health.trial069.stdout:{"epoch":11,"fsid":"4aabec94-a67f-4d24-9557-469e7f74026e","created":"2026-02-20T21:04:58.813036+0000","modified":"2026-02-20T21:05:06.047697+0000","last_up_change":"2026-02-20T21:05:02.821362+0000","last_in_change":"2026-02-20T21:04:59.484260+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:05:03.055498+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"567f2427-666c-49f4-aff5-54e91a2350f8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6800","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6801","nonce":67491597}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6802","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6803","nonce":67491597}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6806","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6807","nonce":67491597}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6804","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6805","nonce":67491597}]},"public_addr":"10.20.193.69:6801/67491597","cluster_addr":"10.20.193.69:6803/67491597","heartbeat_back_addr":"10.20.193.69:6807/67491597","heartbeat_front_addr":"10.20.193.69:6805/67491597","state":["exists","up"]},{"osd":1,"uuid":"c06ca60d-cc95-4882-95d9-c4751ba99875","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6808","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6809","nonce":4090895406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6810","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6811","nonce":4090895406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6814","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6815","nonce":4090895406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6812","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6813","nonce":4090895406}]},"public_addr":"10.20.193.69:6809/4090895406","cluster_addr":"10.20.193.69:6811/4090895406","heartbeat_back_addr":"10.20.193.69:6815/4090895406","heartbeat_front_addr":"10.20.193.69:6813/4090895406","state":["exists","up"]},{"osd":2,"uuid":"6805f378-c47b-4987-8d01-3c839ec7d272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6816","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6817","nonce":1475133124}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6818","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6819","nonce":1475133124}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6822","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6823","nonce":1475133124}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6820","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6821","nonce":1475133124}]},"public_addr":"10.20.193.69:6817/1475133124","cluster_addr":"10.20.193.69:6819/1475133124","heartbeat_back_addr":"10.20.193.69:6823/1475133124","heartbeat_front_addr":"10.20.193.69:6821/1475133124","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:01.438954+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:01.364795+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:01.445158+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:05:06.468 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-20T21:05:06.468 INFO:tasks.ceph:Creating RBD pool 2026-02-20T21:05:06.468 DEBUG:teuthology.orchestra.run.trial069:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-20T21:05:07.021 INFO:teuthology.orchestra.run.trial069.stderr:pool 'rbd' created 2026-02-20T21:05:07.033 DEBUG:teuthology.orchestra.run.trial069:> rbd --cluster cluster1 pool init rbd 2026-02-20T21:05:10.034 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-20T21:05:10.035 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 config log 1 --format=json 2026-02-20T21:05:10.035 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-20T21:05:10.214 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:10.224 INFO:teuthology.orchestra.run.trial069.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-02-20T21:05:10.224 INFO:tasks.ceph_manager:config epoch is 1 2026-02-20T21:05:10.225 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-20T21:05:10.225 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-20T21:05:10.225 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-20T21:05:10.437 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:10.445 INFO:teuthology.orchestra.run.trial069.stdout:{"epoch":5,"flags":0,"active_gid":4104,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6824","nonce":3317095991},{"type":"v1","addr":"10.20.193.69:6825","nonce":3317095991}]},"active_addr":"10.20.193.69:6825/3317095991","active_change":"2026-02-20T21:05:01.040506+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"prometheus_tls_secret_name":{"name":"prometheus_tls_secret_name","type":"str","level":"advanced","flags":0,"default_value":"rook-ceph-prometheus-server-tls","min":"","max":"","enum_allowed":[],"desc":"name of tls secret in k8s for prometheus","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.69:0","nonce":936862588}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.69:0","nonce":1350372519}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.69:0","nonce":4271444806}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.69:0","nonce":1362349519}]}]} 2026-02-20T21:05:10.449 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-20T21:05:10.449 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-20T21:05:10.449 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:05:10.629 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:10.629 INFO:teuthology.orchestra.run.trial069.stdout:{"epoch":15,"fsid":"4aabec94-a67f-4d24-9557-469e7f74026e","created":"2026-02-20T21:04:58.813036+0000","modified":"2026-02-20T21:05:10.055286+0000","last_up_change":"2026-02-20T21:05:02.821362+0000","last_in_change":"2026-02-20T21:04:59.484260+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:05:03.055498+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:05:06.677117+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"567f2427-666c-49f4-aff5-54e91a2350f8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6800","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6801","nonce":67491597}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6802","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6803","nonce":67491597}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6806","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6807","nonce":67491597}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6804","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6805","nonce":67491597}]},"public_addr":"10.20.193.69:6801/67491597","cluster_addr":"10.20.193.69:6803/67491597","heartbeat_back_addr":"10.20.193.69:6807/67491597","heartbeat_front_addr":"10.20.193.69:6805/67491597","state":["exists","up"]},{"osd":1,"uuid":"c06ca60d-cc95-4882-95d9-c4751ba99875","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6808","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6809","nonce":4090895406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6810","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6811","nonce":4090895406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6814","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6815","nonce":4090895406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6812","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6813","nonce":4090895406}]},"public_addr":"10.20.193.69:6809/4090895406","cluster_addr":"10.20.193.69:6811/4090895406","heartbeat_back_addr":"10.20.193.69:6815/4090895406","heartbeat_front_addr":"10.20.193.69:6813/4090895406","state":["exists","up"]},{"osd":2,"uuid":"6805f378-c47b-4987-8d01-3c839ec7d272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6816","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6817","nonce":1475133124}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6818","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6819","nonce":1475133124}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6822","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6823","nonce":1475133124}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6820","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6821","nonce":1475133124}]},"public_addr":"10.20.193.69:6817/1475133124","cluster_addr":"10.20.193.69:6819/1475133124","heartbeat_back_addr":"10.20.193.69:6823/1475133124","heartbeat_front_addr":"10.20.193.69:6821/1475133124","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:01.438954+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:01.364795+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:01.445158+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:05:10.638 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-20T21:05:10.638 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:05:10.818 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:10.819 INFO:teuthology.orchestra.run.trial069.stdout:{"epoch":15,"fsid":"4aabec94-a67f-4d24-9557-469e7f74026e","created":"2026-02-20T21:04:58.813036+0000","modified":"2026-02-20T21:05:10.055286+0000","last_up_change":"2026-02-20T21:05:02.821362+0000","last_in_change":"2026-02-20T21:04:59.484260+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:05:03.055498+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:05:06.677117+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"567f2427-666c-49f4-aff5-54e91a2350f8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6800","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6801","nonce":67491597}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6802","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6803","nonce":67491597}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6806","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6807","nonce":67491597}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6804","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6805","nonce":67491597}]},"public_addr":"10.20.193.69:6801/67491597","cluster_addr":"10.20.193.69:6803/67491597","heartbeat_back_addr":"10.20.193.69:6807/67491597","heartbeat_front_addr":"10.20.193.69:6805/67491597","state":["exists","up"]},{"osd":1,"uuid":"c06ca60d-cc95-4882-95d9-c4751ba99875","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6808","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6809","nonce":4090895406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6810","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6811","nonce":4090895406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6814","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6815","nonce":4090895406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6812","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6813","nonce":4090895406}]},"public_addr":"10.20.193.69:6809/4090895406","cluster_addr":"10.20.193.69:6811/4090895406","heartbeat_back_addr":"10.20.193.69:6815/4090895406","heartbeat_front_addr":"10.20.193.69:6813/4090895406","state":["exists","up"]},{"osd":2,"uuid":"6805f378-c47b-4987-8d01-3c839ec7d272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6816","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6817","nonce":1475133124}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6818","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6819","nonce":1475133124}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6822","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6823","nonce":1475133124}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6820","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6821","nonce":1475133124}]},"public_addr":"10.20.193.69:6817/1475133124","cluster_addr":"10.20.193.69:6819/1475133124","heartbeat_back_addr":"10.20.193.69:6823/1475133124","heartbeat_front_addr":"10.20.193.69:6821/1475133124","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:01.438954+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:01.364795+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:01.445158+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:05:10.828 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-20T21:05:10.828 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-20T21:05:10.829 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-20T21:05:10.918 INFO:teuthology.orchestra.run.trial069.stdout:34359738371 2026-02-20T21:05:10.918 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-20T21:05:10.920 INFO:teuthology.orchestra.run.trial069.stdout:34359738371 2026-02-20T21:05:10.920 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-20T21:05:10.921 INFO:teuthology.orchestra.run.trial069.stdout:34359738371 2026-02-20T21:05:10.921 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-20T21:05:11.143 INFO:teuthology.orchestra.run.trial069.stdout:34359738371 2026-02-20T21:05:11.145 INFO:teuthology.orchestra.run.trial069.stdout:34359738371 2026-02-20T21:05:11.149 INFO:teuthology.orchestra.run.trial069.stdout:34359738371 2026-02-20T21:05:11.155 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.2 2026-02-20T21:05:11.155 DEBUG:teuthology.parallel:result is None 2026-02-20T21:05:11.156 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.1 2026-02-20T21:05:11.157 DEBUG:teuthology.parallel:result is None 2026-02-20T21:05:11.159 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738371 for osd.0 2026-02-20T21:05:11.160 DEBUG:teuthology.parallel:result is None 2026-02-20T21:05:11.160 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-20T21:05:11.160 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:05:11.379 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:11.379 INFO:teuthology.orchestra.run.trial069.stderr:dumped all 2026-02-20T21:05:11.388 INFO:teuthology.orchestra.run.trial069.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2026-02-20T21:05:11.045007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82424,"kb_used_data":1544,"kb_used_omap":23,"kb_used_meta":80808,"kb_avail":283033096,"statfs":{"total":289910292480,"available":289825890304,"internally_reserved":0,"allocated":1581056,"data_stored":1292257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23725,"internal_metadata":82748243},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.992373"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.057681+0000","last_change":"2026-02-20T21:05:10.057921+0000","last_active":"2026-02-20T21:05:10.057681+0000","last_peered":"2026-02-20T21:05:10.057681+0000","last_clean":"2026-02-20T21:05:10.057681+0000","last_became_active":"2026-02-20T21:05:08.055580+0000","last_became_peered":"2026-02-20T21:05:08.055580+0000","last_unstale":"2026-02-20T21:05:10.057681+0000","last_undegraded":"2026-02-20T21:05:10.057681+0000","last_fullsized":"2026-02-20T21:05:10.057681+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:02:05.883424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079302400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.057519+0000","last_change":"2026-02-20T21:05:10.057723+0000","last_active":"2026-02-20T21:05:10.057519+0000","last_peered":"2026-02-20T21:05:10.057519+0000","last_clean":"2026-02-20T21:05:10.057519+0000","last_became_active":"2026-02-20T21:05:08.055678+0000","last_became_peered":"2026-02-20T21:05:08.055678+0000","last_unstale":"2026-02-20T21:05:10.057519+0000","last_undegraded":"2026-02-20T21:05:10.057519+0000","last_fullsized":"2026-02-20T21:05:10.057519+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:13:21.229779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061507100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.057330+0000","last_change":"2026-02-20T21:05:10.057558+0000","last_active":"2026-02-20T21:05:10.057330+0000","last_peered":"2026-02-20T21:05:10.057330+0000","last_clean":"2026-02-20T21:05:10.057330+0000","last_became_active":"2026-02-20T21:05:08.055784+0000","last_became_peered":"2026-02-20T21:05:08.055784+0000","last_unstale":"2026-02-20T21:05:10.057330+0000","last_undegraded":"2026-02-20T21:05:10.057330+0000","last_fullsized":"2026-02-20T21:05:10.057330+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:08:45.110689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046839599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.057758+0000","last_change":"2026-02-20T21:05:10.058070+0000","last_active":"2026-02-20T21:05:10.057758+0000","last_peered":"2026-02-20T21:05:10.057758+0000","last_clean":"2026-02-20T21:05:10.057758+0000","last_became_active":"2026-02-20T21:05:08.055399+0000","last_became_peered":"2026-02-20T21:05:08.055399+0000","last_unstale":"2026-02-20T21:05:10.057758+0000","last_undegraded":"2026-02-20T21:05:10.057758+0000","last_fullsized":"2026-02-20T21:05:10.057758+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:30:58.729968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091594400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.059734+0000","last_change":"2026-02-20T21:05:10.059734+0000","last_active":"2026-02-20T21:05:10.059734+0000","last_peered":"2026-02-20T21:05:10.059734+0000","last_clean":"2026-02-20T21:05:10.059734+0000","last_became_active":"2026-02-20T21:05:08.055463+0000","last_became_peered":"2026-02-20T21:05:08.055463+0000","last_unstale":"2026-02-20T21:05:10.059734+0000","last_undegraded":"2026-02-20T21:05:10.059734+0000","last_fullsized":"2026-02-20T21:05:10.059734+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:01:55.764306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080464600000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.570134+0000","last_change":"2026-02-20T21:05:10.570249+0000","last_active":"2026-02-20T21:05:10.570134+0000","last_peered":"2026-02-20T21:05:10.570134+0000","last_clean":"2026-02-20T21:05:10.570134+0000","last_became_active":"2026-02-20T21:05:08.055414+0000","last_became_peered":"2026-02-20T21:05:08.055414+0000","last_unstale":"2026-02-20T21:05:10.570134+0000","last_undegraded":"2026-02-20T21:05:10.570134+0000","last_fullsized":"2026-02-20T21:05:10.570134+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:02:16.632242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026386399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.570103+0000","last_change":"2026-02-20T21:05:10.570241+0000","last_active":"2026-02-20T21:05:10.570103+0000","last_peered":"2026-02-20T21:05:10.570103+0000","last_clean":"2026-02-20T21:05:10.570103+0000","last_became_active":"2026-02-20T21:05:08.055485+0000","last_became_peered":"2026-02-20T21:05:08.055485+0000","last_unstale":"2026-02-20T21:05:10.570103+0000","last_undegraded":"2026-02-20T21:05:10.570103+0000","last_fullsized":"2026-02-20T21:05:10.570103+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:51:10.827259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027411299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.057300+0000","last_change":"2026-02-20T21:05:10.057464+0000","last_active":"2026-02-20T21:05:10.057300+0000","last_peered":"2026-02-20T21:05:10.057300+0000","last_clean":"2026-02-20T21:05:10.057300+0000","last_became_active":"2026-02-20T21:05:08.055280+0000","last_became_peered":"2026-02-20T21:05:08.055280+0000","last_unstale":"2026-02-20T21:05:10.057300+0000","last_undegraded":"2026-02-20T21:05:10.057300+0000","last_fullsized":"2026-02-20T21:05:10.057300+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:37:04.239908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000404788,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":110,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.056850+0000","last_change":"2026-02-20T21:05:05.374522+0000","last_active":"2026-02-20T21:05:10.056850+0000","last_peered":"2026-02-20T21:05:10.056850+0000","last_clean":"2026-02-20T21:05:10.056850+0000","last_became_active":"2026-02-20T21:05:05.373652+0000","last_became_peered":"2026-02-20T21:05:05.373652+0000","last_unstale":"2026-02-20T21:05:10.056850+0000","last_undegraded":"2026-02-20T21:05:10.056850+0000","last_fullsized":"2026-02-20T21:05:10.056850+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:04.046826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:04.046826+0000","last_clean_scrub_stamp":"2026-02-20T21:05:04.046826+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:34:03.727741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27080,"kb_used_data":120,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344760,"statfs":{"total":96636764160,"available":96609034240,"internally_reserved":0,"allocated":122880,"data_stored":34785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27583181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27672,"kb_used_data":712,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344168,"statfs":{"total":96636764160,"available":96608428032,"internally_reserved":0,"allocated":729088,"data_stored":628736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27672,"kb_used_data":712,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344168,"statfs":{"total":96636764160,"available":96608428032,"internally_reserved":0,"allocated":729088,"data_stored":628736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:05:11.389 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:05:11.571 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:11.571 INFO:teuthology.orchestra.run.trial069.stderr:dumped all 2026-02-20T21:05:11.579 INFO:teuthology.orchestra.run.trial069.stdout:{"pg_ready":true,"pg_map":{"version":16,"stamp":"2026-02-20T21:05:11.045007+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82424,"kb_used_data":1544,"kb_used_omap":23,"kb_used_meta":80808,"kb_avail":283033096,"statfs":{"total":289910292480,"available":289825890304,"internally_reserved":0,"allocated":1581056,"data_stored":1292257,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":23725,"internal_metadata":82748243},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"2.992373"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.057681+0000","last_change":"2026-02-20T21:05:10.057921+0000","last_active":"2026-02-20T21:05:10.057681+0000","last_peered":"2026-02-20T21:05:10.057681+0000","last_clean":"2026-02-20T21:05:10.057681+0000","last_became_active":"2026-02-20T21:05:08.055580+0000","last_became_peered":"2026-02-20T21:05:08.055580+0000","last_unstale":"2026-02-20T21:05:10.057681+0000","last_undegraded":"2026-02-20T21:05:10.057681+0000","last_fullsized":"2026-02-20T21:05:10.057681+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:02:05.883424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079302400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.057519+0000","last_change":"2026-02-20T21:05:10.057723+0000","last_active":"2026-02-20T21:05:10.057519+0000","last_peered":"2026-02-20T21:05:10.057519+0000","last_clean":"2026-02-20T21:05:10.057519+0000","last_became_active":"2026-02-20T21:05:08.055678+0000","last_became_peered":"2026-02-20T21:05:08.055678+0000","last_unstale":"2026-02-20T21:05:10.057519+0000","last_undegraded":"2026-02-20T21:05:10.057519+0000","last_fullsized":"2026-02-20T21:05:10.057519+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:13:21.229779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061507100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.057330+0000","last_change":"2026-02-20T21:05:10.057558+0000","last_active":"2026-02-20T21:05:10.057330+0000","last_peered":"2026-02-20T21:05:10.057330+0000","last_clean":"2026-02-20T21:05:10.057330+0000","last_became_active":"2026-02-20T21:05:08.055784+0000","last_became_peered":"2026-02-20T21:05:08.055784+0000","last_unstale":"2026-02-20T21:05:10.057330+0000","last_undegraded":"2026-02-20T21:05:10.057330+0000","last_fullsized":"2026-02-20T21:05:10.057330+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:08:45.110689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046839599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.057758+0000","last_change":"2026-02-20T21:05:10.058070+0000","last_active":"2026-02-20T21:05:10.057758+0000","last_peered":"2026-02-20T21:05:10.057758+0000","last_clean":"2026-02-20T21:05:10.057758+0000","last_became_active":"2026-02-20T21:05:08.055399+0000","last_became_peered":"2026-02-20T21:05:08.055399+0000","last_unstale":"2026-02-20T21:05:10.057758+0000","last_undegraded":"2026-02-20T21:05:10.057758+0000","last_fullsized":"2026-02-20T21:05:10.057758+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:30:58.729968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091594400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":22,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.059734+0000","last_change":"2026-02-20T21:05:10.059734+0000","last_active":"2026-02-20T21:05:10.059734+0000","last_peered":"2026-02-20T21:05:10.059734+0000","last_clean":"2026-02-20T21:05:10.059734+0000","last_became_active":"2026-02-20T21:05:08.055463+0000","last_became_peered":"2026-02-20T21:05:08.055463+0000","last_unstale":"2026-02-20T21:05:10.059734+0000","last_undegraded":"2026-02-20T21:05:10.059734+0000","last_fullsized":"2026-02-20T21:05:10.059734+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:01:55.764306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080464600000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.570134+0000","last_change":"2026-02-20T21:05:10.570249+0000","last_active":"2026-02-20T21:05:10.570134+0000","last_peered":"2026-02-20T21:05:10.570134+0000","last_clean":"2026-02-20T21:05:10.570134+0000","last_became_active":"2026-02-20T21:05:08.055414+0000","last_became_peered":"2026-02-20T21:05:08.055414+0000","last_unstale":"2026-02-20T21:05:10.570134+0000","last_undegraded":"2026-02-20T21:05:10.570134+0000","last_fullsized":"2026-02-20T21:05:10.570134+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:02:16.632242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026386399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.570103+0000","last_change":"2026-02-20T21:05:10.570241+0000","last_active":"2026-02-20T21:05:10.570103+0000","last_peered":"2026-02-20T21:05:10.570103+0000","last_clean":"2026-02-20T21:05:10.570103+0000","last_became_active":"2026-02-20T21:05:08.055485+0000","last_became_peered":"2026-02-20T21:05:08.055485+0000","last_unstale":"2026-02-20T21:05:10.570103+0000","last_undegraded":"2026-02-20T21:05:10.570103+0000","last_fullsized":"2026-02-20T21:05:10.570103+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:51:10.827259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027411299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.057300+0000","last_change":"2026-02-20T21:05:10.057464+0000","last_active":"2026-02-20T21:05:10.057300+0000","last_peered":"2026-02-20T21:05:10.057300+0000","last_clean":"2026-02-20T21:05:10.057300+0000","last_became_active":"2026-02-20T21:05:08.055280+0000","last_became_peered":"2026-02-20T21:05:08.055280+0000","last_unstale":"2026-02-20T21:05:10.057300+0000","last_undegraded":"2026-02-20T21:05:10.057300+0000","last_fullsized":"2026-02-20T21:05:10.057300+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:37:04.239908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000404788,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"10'72","reported_seq":110,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-20T21:05:10.056850+0000","last_change":"2026-02-20T21:05:05.374522+0000","last_active":"2026-02-20T21:05:10.056850+0000","last_peered":"2026-02-20T21:05:10.056850+0000","last_clean":"2026-02-20T21:05:10.056850+0000","last_became_active":"2026-02-20T21:05:05.373652+0000","last_became_peered":"2026-02-20T21:05:05.373652+0000","last_unstale":"2026-02-20T21:05:10.056850+0000","last_undegraded":"2026-02-20T21:05:10.056850+0000","last_fullsized":"2026-02-20T21:05:10.056850+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:04.046826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:04.046826+0000","last_clean_scrub_stamp":"2026-02-20T21:05:04.046826+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:34:03.727741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27080,"kb_used_data":120,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344760,"statfs":{"total":96636764160,"available":96609034240,"internally_reserved":0,"allocated":122880,"data_stored":34785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7475,"internal_metadata":27583181},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738371,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27672,"kb_used_data":712,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344168,"statfs":{"total":96636764160,"available":96608428032,"internally_reserved":0,"allocated":729088,"data_stored":628736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738371,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27672,"kb_used_data":712,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344168,"statfs":{"total":96636764160,"available":96608428032,"internally_reserved":0,"allocated":729088,"data_stored":628736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8125,"internal_metadata":27582531},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:05:11.581 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-20T21:05:11.581 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-20T21:05:11.581 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-20T21:05:11.581 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-20T21:05:11.787 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:11.787 INFO:teuthology.orchestra.run.trial069.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-20T21:05:11.795 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-20T21:05:11.795 INFO:teuthology.run_tasks:Running task ceph... 2026-02-20T21:05:11.808 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-20T21:05:11.808 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-20T21:05:11.809 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424', 'branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'description': 'rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-journal-stress-workunit}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '61424', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run four rbd-mirror daemons per cluster'}, {'desc': 'run the rbd_mirror_stress.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial', 'no_nested_subset': False, 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-pdonnell-testing-20260219.182737-tentacle'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 7161, 'sha1': 'f7870454149f22574086dd2ff55d050585136531', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-pdonnell-testing-20260219.182737-tentacle', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f7870454149f22574086dd2ff55d050585136531/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f7870454149f22574086dd2ff55d050585136531', 'targets': {'trial069.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEoqTcZenIAF+X4wTlKR02Zcwn/bnCYUvW9fRROi/bwifmBbXs+0+4g9t+pgqRVjFR31ZzHFEE27LW18lMALvwo=', 'trial163.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBI41I1LStMFdfvKEyoHbBOSe5S7OwVwzjasADkZJME4I0YpV+/nK0xUPVNn+RRY2yhe50w+BDaglotviCmH4WSk='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f7870454149f22574086dd2ff55d050585136531'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster1.client.mirror.3', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.0', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.1', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.2', 'thrash': True}}, {'rbd-mirror': {'client': 'cluster2.client.mirror.3', 'thrash': True}}, {'rbd-mirror-thrash': {'cluster': 'cluster1'}}, {'rbd-mirror-thrash': {'cluster': 'cluster2'}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_stress.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'pool', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1', 'RBD_MIRROR_USE_RBD_MIRROR': '1'}, 'timeout': '6h'}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-20_16:15:01', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.1540699'} 2026-02-20T21:05:11.810 DEBUG:teuthology.orchestra.run.trial069:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-20T21:05:11.812 DEBUG:teuthology.orchestra.run.trial163:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-20T21:05:11.831 DEBUG:teuthology.orchestra.run.trial069:> sudo install -d -m0777 -- /var/run/ceph 2026-02-20T21:05:11.869 DEBUG:teuthology.orchestra.run.trial163:> sudo install -d -m0777 -- /var/run/ceph 2026-02-20T21:05:11.897 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:11.897 DEBUG:teuthology.orchestra.run.trial163:> dd if=/scratch_devs of=/dev/stdout 2026-02-20T21:05:11.955 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-20T21:05:11.955 DEBUG:teuthology.orchestra.run.trial163:> stat /dev/vg_nvme/lv_1 2026-02-20T21:05:12.013 INFO:teuthology.orchestra.run.trial163.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-20T21:05:12.014 INFO:teuthology.orchestra.run.trial163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:05:12.014 INFO:teuthology.orchestra.run.trial163.stdout:Device: 6h/6d Inode: 911 Links: 1 2026-02-20T21:05:12.014 INFO:teuthology.orchestra.run.trial163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:05:12.014 INFO:teuthology.orchestra.run.trial163.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:05:12.014 INFO:teuthology.orchestra.run.trial163.stdout:Access: 2026-02-20 21:04:35.431870685 +0000 2026-02-20T21:05:12.014 INFO:teuthology.orchestra.run.trial163.stdout:Modify: 2026-02-20 21:03:13.079125498 +0000 2026-02-20T21:05:12.014 INFO:teuthology.orchestra.run.trial163.stdout:Change: 2026-02-20 21:03:13.079125498 +0000 2026-02-20T21:05:12.014 INFO:teuthology.orchestra.run.trial163.stdout: Birth: 2026-02-20 21:03:13.079125498 +0000 2026-02-20T21:05:12.015 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-20T21:05:12.079 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records in 2026-02-20T21:05:12.079 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records out 2026-02-20T21:05:12.079 INFO:teuthology.orchestra.run.trial163.stderr:512 bytes copied, 0.00016686 s, 3.1 MB/s 2026-02-20T21:05:12.081 DEBUG:teuthology.orchestra.run.trial163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-20T21:05:12.139 DEBUG:teuthology.orchestra.run.trial163:> stat /dev/vg_nvme/lv_2 2026-02-20T21:05:12.196 INFO:teuthology.orchestra.run.trial163.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-20T21:05:12.197 INFO:teuthology.orchestra.run.trial163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:05:12.197 INFO:teuthology.orchestra.run.trial163.stdout:Device: 6h/6d Inode: 937 Links: 1 2026-02-20T21:05:12.197 INFO:teuthology.orchestra.run.trial163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:05:12.197 INFO:teuthology.orchestra.run.trial163.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:05:12.197 INFO:teuthology.orchestra.run.trial163.stdout:Access: 2026-02-20 21:04:35.431870685 +0000 2026-02-20T21:05:12.197 INFO:teuthology.orchestra.run.trial163.stdout:Modify: 2026-02-20 21:03:13.325115886 +0000 2026-02-20T21:05:12.197 INFO:teuthology.orchestra.run.trial163.stdout:Change: 2026-02-20 21:03:13.325115886 +0000 2026-02-20T21:05:12.197 INFO:teuthology.orchestra.run.trial163.stdout: Birth: 2026-02-20 21:03:13.325115886 +0000 2026-02-20T21:05:12.198 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-20T21:05:12.259 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records in 2026-02-20T21:05:12.259 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records out 2026-02-20T21:05:12.259 INFO:teuthology.orchestra.run.trial163.stderr:512 bytes copied, 0.000144868 s, 3.5 MB/s 2026-02-20T21:05:12.260 DEBUG:teuthology.orchestra.run.trial163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-20T21:05:12.316 DEBUG:teuthology.orchestra.run.trial163:> stat /dev/vg_nvme/lv_3 2026-02-20T21:05:12.372 INFO:teuthology.orchestra.run.trial163.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-20T21:05:12.372 INFO:teuthology.orchestra.run.trial163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:05:12.373 INFO:teuthology.orchestra.run.trial163.stdout:Device: 6h/6d Inode: 966 Links: 1 2026-02-20T21:05:12.373 INFO:teuthology.orchestra.run.trial163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:05:12.373 INFO:teuthology.orchestra.run.trial163.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:05:12.373 INFO:teuthology.orchestra.run.trial163.stdout:Access: 2026-02-20 21:04:35.431870685 +0000 2026-02-20T21:05:12.373 INFO:teuthology.orchestra.run.trial163.stdout:Modify: 2026-02-20 21:03:13.563106587 +0000 2026-02-20T21:05:12.373 INFO:teuthology.orchestra.run.trial163.stdout:Change: 2026-02-20 21:03:13.563106587 +0000 2026-02-20T21:05:12.373 INFO:teuthology.orchestra.run.trial163.stdout: Birth: 2026-02-20 21:03:13.563106587 +0000 2026-02-20T21:05:12.374 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-20T21:05:12.436 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records in 2026-02-20T21:05:12.437 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records out 2026-02-20T21:05:12.437 INFO:teuthology.orchestra.run.trial163.stderr:512 bytes copied, 0.000146031 s, 3.5 MB/s 2026-02-20T21:05:12.438 DEBUG:teuthology.orchestra.run.trial163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-20T21:05:12.495 DEBUG:teuthology.orchestra.run.trial163:> stat /dev/vg_nvme/lv_4 2026-02-20T21:05:12.552 INFO:teuthology.orchestra.run.trial163.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-20T21:05:12.552 INFO:teuthology.orchestra.run.trial163.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-20T21:05:12.552 INFO:teuthology.orchestra.run.trial163.stdout:Device: 6h/6d Inode: 993 Links: 1 2026-02-20T21:05:12.552 INFO:teuthology.orchestra.run.trial163.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-20T21:05:12.552 INFO:teuthology.orchestra.run.trial163.stdout:Context: system_u:object_r:device_t:s0 2026-02-20T21:05:12.552 INFO:teuthology.orchestra.run.trial163.stdout:Access: 2026-02-20 21:04:35.432870641 +0000 2026-02-20T21:05:12.552 INFO:teuthology.orchestra.run.trial163.stdout:Modify: 2026-02-20 21:03:13.808097016 +0000 2026-02-20T21:05:12.552 INFO:teuthology.orchestra.run.trial163.stdout:Change: 2026-02-20 21:03:13.808097016 +0000 2026-02-20T21:05:12.552 INFO:teuthology.orchestra.run.trial163.stdout: Birth: 2026-02-20 21:03:13.808097016 +0000 2026-02-20T21:05:12.553 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-20T21:05:12.616 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records in 2026-02-20T21:05:12.616 INFO:teuthology.orchestra.run.trial163.stderr:1+0 records out 2026-02-20T21:05:12.617 INFO:teuthology.orchestra.run.trial163.stderr:512 bytes copied, 0.000144758 s, 3.5 MB/s 2026-02-20T21:05:12.618 DEBUG:teuthology.orchestra.run.trial163:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-20T21:05:12.676 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:05:12.676 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial163.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-20T21:05:12.677 INFO:tasks.ceph:Generating config... 2026-02-20T21:05:12.679 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-20T21:05:12.679 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-20T21:05:12.679 INFO:tasks.ceph:[client] log to stderr = False 2026-02-20T21:05:12.679 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-20T21:05:12.679 INFO:tasks.ceph:[client] rbd mirror image policy type = simple 2026-02-20T21:05:12.679 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:05:12.679 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:05:12.680 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:05:12.680 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:05:12.680 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:05:12.680 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:05:12.680 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:05:12.680 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:05:12.680 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-20T21:05:12.680 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-20T21:05:12.680 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-20T21:05:12.680 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-20T21:05:12.680 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-20T21:05:12.681 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-20T21:05:12.681 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-20T21:05:12.681 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-20T21:05:12.681 INFO:tasks.ceph:[osd] bdev async discard = True 2026-02-20T21:05:12.681 INFO:tasks.ceph:[osd] bdev enable discard = True 2026-02-20T21:05:12.681 INFO:tasks.ceph:[osd] bluestore allocator = stupid 2026-02-20T21:05:12.681 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-20T21:05:12.681 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-20T21:05:12.681 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-20T21:05:12.681 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-20T21:05:12.681 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-20T21:05:12.682 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-20T21:05:12.682 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-20T21:05:12.682 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-20T21:05:12.682 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-20T21:05:12.682 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-20T21:05:12.682 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-20T21:05:12.682 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-20T21:05:12.682 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-20T21:05:12.682 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-20T21:05:12.751 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.keyring 2026-02-20T21:05:12.754 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-20T21:05:12.830 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-20T21:05:12.896 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.163')] 2026-02-20T21:05:12.896 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.163', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'simple'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-20T21:05:12.897 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:12.898 DEBUG:teuthology.orchestra.run.trial163:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-20T21:05:12.953 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.163 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:05:13.027 INFO:teuthology.orchestra.run.trial163.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:05:13.027 INFO:teuthology.orchestra.run.trial163.stdout:monmaptool: generated fsid 6b7cbdea-dd31-4132-aa0e-90dc10bb8129 2026-02-20T21:05:13.027 INFO:teuthology.orchestra.run.trial163.stdout:setting min_mon_release = tentacle 2026-02-20T21:05:13.027 INFO:teuthology.orchestra.run.trial163.stdout:epoch 0 2026-02-20T21:05:13.028 INFO:teuthology.orchestra.run.trial163.stdout:fsid 6b7cbdea-dd31-4132-aa0e-90dc10bb8129 2026-02-20T21:05:13.028 INFO:teuthology.orchestra.run.trial163.stdout:last_changed 2026-02-20T21:05:13.026355+0000 2026-02-20T21:05:13.028 INFO:teuthology.orchestra.run.trial163.stdout:created 2026-02-20T21:05:13.026355+0000 2026-02-20T21:05:13.028 INFO:teuthology.orchestra.run.trial163.stdout:min_mon_release 20 (tentacle) 2026-02-20T21:05:13.028 INFO:teuthology.orchestra.run.trial163.stdout:election_strategy: 1 2026-02-20T21:05:13.028 INFO:teuthology.orchestra.run.trial163.stdout:0: [v2:10.20.193.163:3300/0,v1:10.20.193.163:6789/0] mon.a 2026-02-20T21:05:13.028 INFO:teuthology.orchestra.run.trial163.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-20T21:05:13.030 DEBUG:teuthology.orchestra.run.trial163:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-20T21:05:13.088 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 6b7cbdea-dd31-4132-aa0e-90dc10bb8129... 2026-02-20T21:05:13.090 DEBUG:teuthology.orchestra.run.trial069:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-20T21:05:13.092 DEBUG:teuthology.orchestra.run.trial163:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-20T21:05:13.131 INFO:teuthology.orchestra.run.trial069.stdout:[global] 2026-02-20T21:05:13.132 INFO:teuthology.orchestra.run.trial069.stdout: chdir = "" 2026-02-20T21:05:13.132 INFO:teuthology.orchestra.run.trial069.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-20T21:05:13.132 INFO:teuthology.orchestra.run.trial069.stdout: auth supported = cephx 2026-02-20T21:05:13.132 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.132 INFO:teuthology.orchestra.run.trial069.stdout: filestore xattr use omap = true 2026-02-20T21:05:13.132 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.132 INFO:teuthology.orchestra.run.trial069.stdout: mon clock drift allowed = 1.000 2026-02-20T21:05:13.132 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.133 INFO:teuthology.orchestra.run.trial069.stdout: osd crush chooseleaf type = 0 2026-02-20T21:05:13.133 INFO:teuthology.orchestra.run.trial069.stdout: auth debug = true 2026-02-20T21:05:13.133 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.133 INFO:teuthology.orchestra.run.trial069.stdout: ms die on old message = true 2026-02-20T21:05:13.133 INFO:teuthology.orchestra.run.trial069.stdout: ms die on bug = true 2026-02-20T21:05:13.133 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.133 INFO:teuthology.orchestra.run.trial069.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-20T21:05:13.133 INFO:teuthology.orchestra.run.trial069.stdout: mon pg warn max object skew = 0 2026-02-20T21:05:13.133 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.133 INFO:teuthology.orchestra.run.trial069.stdout: # disable pg_autoscaler by default for new pools 2026-02-20T21:05:13.134 INFO:teuthology.orchestra.run.trial069.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-20T21:05:13.134 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.134 INFO:teuthology.orchestra.run.trial069.stdout: osd pool default size = 2 2026-02-20T21:05:13.134 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.134 INFO:teuthology.orchestra.run.trial069.stdout: mon osd allow primary affinity = true 2026-02-20T21:05:13.134 INFO:teuthology.orchestra.run.trial069.stdout: mon osd allow pg remap = true 2026-02-20T21:05:13.134 INFO:teuthology.orchestra.run.trial069.stdout: mon warn on legacy crush tunables = false 2026-02-20T21:05:13.134 INFO:teuthology.orchestra.run.trial069.stdout: mon warn on crush straw calc version zero = false 2026-02-20T21:05:13.134 INFO:teuthology.orchestra.run.trial069.stdout: mon warn on no sortbitwise = false 2026-02-20T21:05:13.134 INFO:teuthology.orchestra.run.trial069.stdout: mon warn on osd down out interval zero = false 2026-02-20T21:05:13.134 INFO:teuthology.orchestra.run.trial069.stdout: mon warn on too few osds = false 2026-02-20T21:05:13.134 INFO:teuthology.orchestra.run.trial069.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-20T21:05:13.135 INFO:teuthology.orchestra.run.trial069.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-20T21:05:13.135 INFO:teuthology.orchestra.run.trial069.stdout: mon_allow_pool_size_one = true 2026-02-20T21:05:13.135 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.135 INFO:teuthology.orchestra.run.trial069.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-20T21:05:13.135 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.135 INFO:teuthology.orchestra.run.trial069.stdout: osd default data pool replay window = 5 2026-02-20T21:05:13.135 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.135 INFO:teuthology.orchestra.run.trial069.stdout: mon allow pool delete = true 2026-02-20T21:05:13.135 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.135 INFO:teuthology.orchestra.run.trial069.stdout: mon cluster log file level = debug 2026-02-20T21:05:13.135 INFO:teuthology.orchestra.run.trial069.stdout: debug asserts on shutdown = true 2026-02-20T21:05:13.135 INFO:teuthology.orchestra.run.trial069.stdout: mon health detail to clog = false 2026-02-20T21:05:13.136 INFO:teuthology.orchestra.run.trial069.stdout: mon host = 10.20.193.163 2026-02-20T21:05:13.136 INFO:teuthology.orchestra.run.trial069.stdout: mon client directed command retry = 5 2026-02-20T21:05:13.136 INFO:teuthology.orchestra.run.trial069.stdout: mon warn on pool no app = False 2026-02-20T21:05:13.136 INFO:teuthology.orchestra.run.trial069.stdout: ms inject socket failures = 5000 2026-02-20T21:05:13.136 INFO:teuthology.orchestra.run.trial069.stdout: fsid = 6b7cbdea-dd31-4132-aa0e-90dc10bb8129 2026-02-20T21:05:13.136 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.136 INFO:teuthology.orchestra.run.trial069.stdout:[osd] 2026-02-20T21:05:13.136 INFO:teuthology.orchestra.run.trial069.stdout: osd journal size = 100 2026-02-20T21:05:13.136 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.136 INFO:teuthology.orchestra.run.trial069.stdout: osd scrub load threshold = 5.0 2026-02-20T21:05:13.136 INFO:teuthology.orchestra.run.trial069.stdout: osd scrub max interval = 600 2026-02-20T21:05:13.136 INFO:teuthology.orchestra.run.trial069.stdout: osd mclock profile = high_recovery_ops 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: osd mclock skip benchmark = true 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: osd recover clone overlap = true 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: osd recovery max chunk = 1048576 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: osd debug shutdown = true 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: osd debug op order = true 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: osd debug verify stray on activate = true 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: osd debug trim objects = true 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: osd open classes on start = true 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: osd debug pg log writeout = true 2026-02-20T21:05:13.137 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.138 INFO:teuthology.orchestra.run.trial069.stdout: osd deep scrub update digest min age = 30 2026-02-20T21:05:13.138 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.138 INFO:teuthology.orchestra.run.trial069.stdout: osd map max advance = 10 2026-02-20T21:05:13.138 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.138 INFO:teuthology.orchestra.run.trial069.stdout: journal zero on create = true 2026-02-20T21:05:13.138 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.138 INFO:teuthology.orchestra.run.trial069.stdout: filestore ondisk finisher threads = 3 2026-02-20T21:05:13.138 INFO:teuthology.orchestra.run.trial069.stdout: filestore apply finisher threads = 3 2026-02-20T21:05:13.138 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.138 INFO:teuthology.orchestra.run.trial069.stdout: bdev debug aio = true 2026-02-20T21:05:13.138 INFO:teuthology.orchestra.run.trial069.stdout: osd debug misdirected ops = true 2026-02-20T21:05:13.138 INFO:teuthology.orchestra.run.trial069.stdout: bdev async discard = True 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: bdev enable discard = True 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: bluestore allocator = stupid 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: bluestore block size = 96636764160 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: bluestore fsck on mount = True 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: debug bluefs = 1/20 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: debug bluestore = 1/20 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: debug ms = 1 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: debug osd = 20 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: debug rocksdb = 4/10 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: mon osd backfillfull_ratio = 0.85 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: mon osd full ratio = 0.9 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: mon osd nearfull ratio = 0.8 2026-02-20T21:05:13.139 INFO:teuthology.orchestra.run.trial069.stdout: osd failsafe full ratio = 0.95 2026-02-20T21:05:13.140 INFO:teuthology.orchestra.run.trial069.stdout: osd objectstore = bluestore 2026-02-20T21:05:13.140 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.140 INFO:teuthology.orchestra.run.trial069.stdout:[mgr] 2026-02-20T21:05:13.140 INFO:teuthology.orchestra.run.trial069.stdout: debug ms = 1 2026-02-20T21:05:13.140 INFO:teuthology.orchestra.run.trial069.stdout: debug mgr = 20 2026-02-20T21:05:13.140 INFO:teuthology.orchestra.run.trial069.stdout: debug mon = 20 2026-02-20T21:05:13.140 INFO:teuthology.orchestra.run.trial069.stdout: debug auth = 20 2026-02-20T21:05:13.140 INFO:teuthology.orchestra.run.trial069.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:05:13.140 INFO:teuthology.orchestra.run.trial069.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:05:13.140 INFO:teuthology.orchestra.run.trial069.stdout: mgr/telemetry/nag = false 2026-02-20T21:05:13.140 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.140 INFO:teuthology.orchestra.run.trial069.stdout:[mon] 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: debug ms = 1 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: debug mon = 20 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: debug paxos = 20 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: debug auth = 20 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: mon data avail warn = 5 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: mon mgr mkfs grace = 240 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: mon osd reporter subtree level = osd 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: mon osd prime pg temp = true 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-20T21:05:13.141 INFO:teuthology.orchestra.run.trial069.stdout: auth mon ticket ttl = 660 # 11m 2026-02-20T21:05:13.142 INFO:teuthology.orchestra.run.trial069.stdout: auth service ticket ttl = 240 # 4m 2026-02-20T21:05:13.142 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.142 INFO:teuthology.orchestra.run.trial069.stdout: # don't complain about insecure global_id in the test suite 2026-02-20T21:05:13.142 INFO:teuthology.orchestra.run.trial069.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-20T21:05:13.142 INFO:teuthology.orchestra.run.trial069.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-20T21:05:13.142 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.142 INFO:teuthology.orchestra.run.trial069.stdout: # 1m isn't quite enough 2026-02-20T21:05:13.142 INFO:teuthology.orchestra.run.trial069.stdout: mon_down_mkfs_grace = 2m 2026-02-20T21:05:13.142 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.142 INFO:teuthology.orchestra.run.trial069.stdout: mon_warn_on_filestore_osds = false 2026-02-20T21:05:13.142 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:05:13.142 INFO:teuthology.orchestra.run.trial069.stdout:[client] 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout: rgw cache enabled = true 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout: rgw enable ops log = true 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout: rgw enable usage log = true 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout: debug rbd = 20 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout: debug rbd_mirror = 15 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout: log to stderr = False 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout: rbd default features = 125 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout: rbd mirror image policy type = simple 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout:[mon.a] 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.0] 2026-02-20T21:05:13.143 INFO:teuthology.orchestra.run.trial069.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:05:13.144 INFO:teuthology.orchestra.run.trial069.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:05:13.144 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.1] 2026-02-20T21:05:13.144 INFO:teuthology.orchestra.run.trial069.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:05:13.144 INFO:teuthology.orchestra.run.trial069.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:05:13.144 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.2] 2026-02-20T21:05:13.144 INFO:teuthology.orchestra.run.trial069.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:05:13.144 INFO:teuthology.orchestra.run.trial069.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:05:13.144 INFO:teuthology.orchestra.run.trial069.stdout:[client.mirror.3] 2026-02-20T21:05:13.144 INFO:teuthology.orchestra.run.trial069.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:05:13.144 INFO:teuthology.orchestra.run.trial069.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:05:13.170 INFO:teuthology.orchestra.run.trial163.stdout:[global] 2026-02-20T21:05:13.171 INFO:teuthology.orchestra.run.trial163.stdout: chdir = "" 2026-02-20T21:05:13.171 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-20T21:05:13.171 INFO:teuthology.orchestra.run.trial163.stdout: auth supported = cephx 2026-02-20T21:05:13.171 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.171 INFO:teuthology.orchestra.run.trial163.stdout: filestore xattr use omap = true 2026-02-20T21:05:13.171 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.171 INFO:teuthology.orchestra.run.trial163.stdout: mon clock drift allowed = 1.000 2026-02-20T21:05:13.171 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.171 INFO:teuthology.orchestra.run.trial163.stdout: osd crush chooseleaf type = 0 2026-02-20T21:05:13.171 INFO:teuthology.orchestra.run.trial163.stdout: auth debug = true 2026-02-20T21:05:13.171 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.172 INFO:teuthology.orchestra.run.trial163.stdout: ms die on old message = true 2026-02-20T21:05:13.172 INFO:teuthology.orchestra.run.trial163.stdout: ms die on bug = true 2026-02-20T21:05:13.172 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.172 INFO:teuthology.orchestra.run.trial163.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-20T21:05:13.172 INFO:teuthology.orchestra.run.trial163.stdout: mon pg warn max object skew = 0 2026-02-20T21:05:13.172 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.172 INFO:teuthology.orchestra.run.trial163.stdout: # disable pg_autoscaler by default for new pools 2026-02-20T21:05:13.172 INFO:teuthology.orchestra.run.trial163.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-20T21:05:13.172 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.172 INFO:teuthology.orchestra.run.trial163.stdout: osd pool default size = 2 2026-02-20T21:05:13.172 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.172 INFO:teuthology.orchestra.run.trial163.stdout: mon osd allow primary affinity = true 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: mon osd allow pg remap = true 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on legacy crush tunables = false 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on crush straw calc version zero = false 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on no sortbitwise = false 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on osd down out interval zero = false 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on too few osds = false 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: mon_allow_pool_size_one = true 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: osd pool default erasure code profile = plugin=isa technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.173 INFO:teuthology.orchestra.run.trial163.stdout: osd default data pool replay window = 5 2026-02-20T21:05:13.174 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.174 INFO:teuthology.orchestra.run.trial163.stdout: mon allow pool delete = true 2026-02-20T21:05:13.174 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.174 INFO:teuthology.orchestra.run.trial163.stdout: mon cluster log file level = debug 2026-02-20T21:05:13.174 INFO:teuthology.orchestra.run.trial163.stdout: debug asserts on shutdown = true 2026-02-20T21:05:13.174 INFO:teuthology.orchestra.run.trial163.stdout: mon health detail to clog = false 2026-02-20T21:05:13.174 INFO:teuthology.orchestra.run.trial163.stdout: mon host = 10.20.193.163 2026-02-20T21:05:13.174 INFO:teuthology.orchestra.run.trial163.stdout: mon client directed command retry = 5 2026-02-20T21:05:13.174 INFO:teuthology.orchestra.run.trial163.stdout: mon warn on pool no app = False 2026-02-20T21:05:13.174 INFO:teuthology.orchestra.run.trial163.stdout: ms inject socket failures = 5000 2026-02-20T21:05:13.174 INFO:teuthology.orchestra.run.trial163.stdout: fsid = 6b7cbdea-dd31-4132-aa0e-90dc10bb8129 2026-02-20T21:05:13.174 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout:[osd] 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout: osd journal size = 100 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout: osd scrub load threshold = 5.0 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout: osd scrub max interval = 600 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout: osd mclock profile = high_recovery_ops 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout: osd mclock skip benchmark = true 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout: osd recover clone overlap = true 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout: osd recovery max chunk = 1048576 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout: osd debug shutdown = true 2026-02-20T21:05:13.175 INFO:teuthology.orchestra.run.trial163.stdout: osd debug op order = true 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: osd debug verify stray on activate = true 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: osd debug trim objects = true 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: osd open classes on start = true 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: osd debug pg log writeout = true 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: osd deep scrub update digest min age = 30 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: osd map max advance = 10 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: journal zero on create = true 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.176 INFO:teuthology.orchestra.run.trial163.stdout: filestore ondisk finisher threads = 3 2026-02-20T21:05:13.177 INFO:teuthology.orchestra.run.trial163.stdout: filestore apply finisher threads = 3 2026-02-20T21:05:13.177 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.177 INFO:teuthology.orchestra.run.trial163.stdout: bdev debug aio = true 2026-02-20T21:05:13.177 INFO:teuthology.orchestra.run.trial163.stdout: osd debug misdirected ops = true 2026-02-20T21:05:13.177 INFO:teuthology.orchestra.run.trial163.stdout: bdev async discard = True 2026-02-20T21:05:13.177 INFO:teuthology.orchestra.run.trial163.stdout: bdev enable discard = True 2026-02-20T21:05:13.177 INFO:teuthology.orchestra.run.trial163.stdout: bluestore allocator = stupid 2026-02-20T21:05:13.177 INFO:teuthology.orchestra.run.trial163.stdout: bluestore block size = 96636764160 2026-02-20T21:05:13.177 INFO:teuthology.orchestra.run.trial163.stdout: bluestore fsck on mount = True 2026-02-20T21:05:13.177 INFO:teuthology.orchestra.run.trial163.stdout: debug bluefs = 1/20 2026-02-20T21:05:13.177 INFO:teuthology.orchestra.run.trial163.stdout: debug bluestore = 1/20 2026-02-20T21:05:13.177 INFO:teuthology.orchestra.run.trial163.stdout: debug ms = 1 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout: debug osd = 20 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout: debug rocksdb = 4/10 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout: mon osd backfillfull_ratio = 0.85 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout: mon osd full ratio = 0.9 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout: mon osd nearfull ratio = 0.8 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout: osd failsafe full ratio = 0.95 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout: osd objectstore = bluestore 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout:[mgr] 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout: debug ms = 1 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout: debug mgr = 20 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout: debug mon = 20 2026-02-20T21:05:13.178 INFO:teuthology.orchestra.run.trial163.stdout: debug auth = 20 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout: mgr/telemetry/nag = false 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout:[mon] 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout: debug ms = 1 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout: debug mon = 20 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout: debug paxos = 20 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout: debug auth = 20 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout: mon data avail warn = 5 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout: mon mgr mkfs grace = 240 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min pgs per osd = 4 2026-02-20T21:05:13.179 INFO:teuthology.orchestra.run.trial163.stdout: mon osd reporter subtree level = osd 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: mon osd prime pg temp = true 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: mon reweight min bytes per osd = 10 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: auth mon ticket ttl = 660 # 11m 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: auth service ticket ttl = 240 # 4m 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: # don't complain about insecure global_id in the test suite 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: # 1m isn't quite enough 2026-02-20T21:05:13.180 INFO:teuthology.orchestra.run.trial163.stdout: mon_down_mkfs_grace = 2m 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout: mon_warn_on_filestore_osds = false 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout:[client] 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout: rgw cache enabled = true 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout: rgw enable ops log = true 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout: rgw enable usage log = true 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout: debug rbd = 20 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout: debug rbd_mirror = 15 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout: log to stderr = False 2026-02-20T21:05:13.181 INFO:teuthology.orchestra.run.trial163.stdout: rbd default features = 125 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout: rbd mirror image policy type = simple 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout:[mon.a] 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.0] 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.1] 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.2] 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.3] 2026-02-20T21:05:13.182 INFO:teuthology.orchestra.run.trial163.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-20T21:05:13.183 INFO:teuthology.orchestra.run.trial163.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-20T21:05:13.184 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-20T21:05:13.184 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-20T21:05:13.266 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-20T21:05:13.266 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:13.267 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-20T21:05:13.322 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:13.322 DEBUG:teuthology.orchestra.run.trial163:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-20T21:05:13.379 INFO:tasks.ceph:Sending monmap to node ubuntu@trial069.front.sepia.ceph.com 2026-02-20T21:05:13.379 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:05:13.379 DEBUG:teuthology.orchestra.run.trial069:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-20T21:05:13.379 DEBUG:teuthology.orchestra.run.trial069:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-20T21:05:13.413 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:05:13.413 DEBUG:teuthology.orchestra.run.trial069:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:05:13.470 INFO:tasks.ceph:Sending monmap to node ubuntu@trial163.front.sepia.ceph.com 2026-02-20T21:05:13.470 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:13.470 DEBUG:teuthology.orchestra.run.trial163:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-20T21:05:13.470 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-20T21:05:13.505 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:13.505 DEBUG:teuthology.orchestra.run.trial163:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:05:13.560 INFO:tasks.ceph:Setting up mon nodes... 2026-02-20T21:05:13.560 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-20T21:05:13.560 DEBUG:teuthology.orchestra.run.trial069:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-20T21:05:13.603 INFO:teuthology.orchestra.run.trial069.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-20T21:05:13.606 INFO:tasks.ceph:Setting up mds nodes... 2026-02-20T21:05:13.606 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-20T21:05:13.606 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-20T21:05:13.680 INFO:teuthology.orchestra.run.trial069.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-20T21:05:13.691 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-20T21:05:13.728 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-20T21:05:13.740 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-20T21:05:13.816 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-20T21:05:13.827 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-20T21:05:13.904 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-20T21:05:13.915 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-20T21:05:13.990 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-20T21:05:14.002 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-20T21:05:14.079 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-20T21:05:14.090 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-20T21:05:14.165 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-20T21:05:14.176 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-20T21:05:14.251 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-20T21:05:14.262 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-20T21:05:14.335 INFO:teuthology.orchestra.run.trial163.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-20T21:05:14.347 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-20T21:05:14.347 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial069.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial163.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-20T21:05:14.348 DEBUG:teuthology.orchestra.run.trial163:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-20T21:05:14.411 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:05:14.412 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-20T21:05:14.412 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial163.front.sepia.ceph.com 2026-02-20T21:05:14.412 DEBUG:teuthology.orchestra.run.trial163:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-20T21:05:14.476 INFO:teuthology.orchestra.run.trial163.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:05:14.477 INFO:teuthology.orchestra.run.trial163.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:05:14.477 INFO:teuthology.orchestra.run.trial163.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:05:14.477 INFO:teuthology.orchestra.run.trial163.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-20T21:05:14.477 INFO:teuthology.orchestra.run.trial163.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:05:14.477 INFO:teuthology.orchestra.run.trial163.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:05:14.477 INFO:teuthology.orchestra.run.trial163.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:05:14.477 INFO:teuthology.orchestra.run.trial163.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:05:14.478 INFO:teuthology.orchestra.run.trial163.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:05:14.478 INFO:teuthology.orchestra.run.trial163.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:05:15.082 INFO:teuthology.orchestra.run.trial163.stdout:Discarding blocks...Done. 2026-02-20T21:05:15.084 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial163.front.sepia.ceph.com -o noatime 2026-02-20T21:05:15.084 DEBUG:teuthology.orchestra.run.trial163:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-20T21:05:15.152 DEBUG:teuthology.orchestra.run.trial163:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-20T21:05:15.219 DEBUG:teuthology.orchestra.run.trial163:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-20T21:05:15.284 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:05:15.284 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-20T21:05:15.285 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial163.front.sepia.ceph.com 2026-02-20T21:05:15.285 DEBUG:teuthology.orchestra.run.trial163:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-20T21:05:15.350 INFO:teuthology.orchestra.run.trial163.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:05:15.351 INFO:teuthology.orchestra.run.trial163.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:05:15.351 INFO:teuthology.orchestra.run.trial163.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:05:15.351 INFO:teuthology.orchestra.run.trial163.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-20T21:05:15.351 INFO:teuthology.orchestra.run.trial163.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:05:15.351 INFO:teuthology.orchestra.run.trial163.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:05:15.351 INFO:teuthology.orchestra.run.trial163.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:05:15.351 INFO:teuthology.orchestra.run.trial163.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:05:15.351 INFO:teuthology.orchestra.run.trial163.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:05:15.351 INFO:teuthology.orchestra.run.trial163.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:05:15.910 INFO:teuthology.orchestra.run.trial163.stdout:Discarding blocks...Done. 2026-02-20T21:05:15.912 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial163.front.sepia.ceph.com -o noatime 2026-02-20T21:05:15.912 DEBUG:teuthology.orchestra.run.trial163:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-20T21:05:15.979 DEBUG:teuthology.orchestra.run.trial163:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-20T21:05:16.046 DEBUG:teuthology.orchestra.run.trial163:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-20T21:05:16.111 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-20T21:05:16.111 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-20T21:05:16.111 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial163.front.sepia.ceph.com 2026-02-20T21:05:16.111 DEBUG:teuthology.orchestra.run.trial163:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-20T21:05:16.176 INFO:teuthology.orchestra.run.trial163.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-20T21:05:16.176 INFO:teuthology.orchestra.run.trial163.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-20T21:05:16.176 INFO:teuthology.orchestra.run.trial163.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-20T21:05:16.176 INFO:teuthology.orchestra.run.trial163.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-20T21:05:16.177 INFO:teuthology.orchestra.run.trial163.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-20T21:05:16.177 INFO:teuthology.orchestra.run.trial163.stdout: = sunit=0 swidth=0 blks 2026-02-20T21:05:16.177 INFO:teuthology.orchestra.run.trial163.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-20T21:05:16.177 INFO:teuthology.orchestra.run.trial163.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-20T21:05:16.177 INFO:teuthology.orchestra.run.trial163.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-20T21:05:16.177 INFO:teuthology.orchestra.run.trial163.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-20T21:05:16.773 INFO:teuthology.orchestra.run.trial163.stdout:Discarding blocks...Done. 2026-02-20T21:05:16.775 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial163.front.sepia.ceph.com -o noatime 2026-02-20T21:05:16.775 DEBUG:teuthology.orchestra.run.trial163:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-20T21:05:16.843 DEBUG:teuthology.orchestra.run.trial163:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-20T21:05:16.910 DEBUG:teuthology.orchestra.run.trial163:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:05:16.988 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-20T21:05:16.986+0000 7ff0a9a10900 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-20T21:05:16.988 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-20T21:05:16.986+0000 7ff0a9a10900 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-20T21:05:16.988 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-20T21:05:16.987+0000 7ff0a9a10900 -1 bdev(0x55e5456c5800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-20T21:05:16.988 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-20T21:05:16.987+0000 7ff0a9a10900 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-20T21:05:17.460 DEBUG:teuthology.orchestra.run.trial163:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-20T21:05:17.525 DEBUG:teuthology.orchestra.run.trial163:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:05:17.603 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-20T21:05:17.601+0000 7fbe58bf0900 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-20T21:05:17.603 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-20T21:05:17.601+0000 7fbe58bf0900 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-20T21:05:17.603 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-20T21:05:17.601+0000 7fbe58bf0900 -1 bdev(0x55793ebbd800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-20T21:05:17.603 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-20T21:05:17.602+0000 7fbe58bf0900 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-20T21:05:18.046 DEBUG:teuthology.orchestra.run.trial163:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-20T21:05:18.111 DEBUG:teuthology.orchestra.run.trial163:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:05:18.192 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-20T21:05:18.191+0000 7f9f7431a900 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-20T21:05:18.192 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-20T21:05:18.191+0000 7f9f7431a900 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-20T21:05:18.193 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-20T21:05:18.191+0000 7f9f7431a900 -1 bdev(0x564786f95800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-20T21:05:18.193 INFO:teuthology.orchestra.run.trial163.stderr:2026-02-20T21:05:18.191+0000 7f9f7431a900 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-20T21:05:18.659 DEBUG:teuthology.orchestra.run.trial163:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-20T21:05:18.723 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-20T21:05:18.723 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:05:18.723 DEBUG:teuthology.orchestra.run.trial069:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-20T21:05:18.749 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:18.749 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-20T21:05:18.790 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:18.790 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-20T21:05:18.854 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:18.854 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-20T21:05:18.917 DEBUG:teuthology.orchestra.run.trial069:> set -ex 2026-02-20T21:05:18.917 DEBUG:teuthology.orchestra.run.trial069:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-20T21:05:18.934 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:18.934 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-20T21:05:18.973 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:18.973 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-20T21:05:19.028 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:19.028 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-20T21:05:19.084 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:19.084 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-20T21:05:19.139 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:19.139 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-20T21:05:19.194 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:19.194 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-20T21:05:19.249 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:19.249 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-20T21:05:19.305 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:19.306 DEBUG:teuthology.orchestra.run.trial163:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-20T21:05:19.362 INFO:tasks.ceph:Adding keys to all mons... 2026-02-20T21:05:19.362 DEBUG:teuthology.orchestra.run.trial163:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-20T21:05:19.424 INFO:teuthology.orchestra.run.trial163.stdout:[mgr.x] 2026-02-20T21:05:19.424 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCJzJhp7mbEJRAA3yVMxLdrsK4FPv/745uowA== 2026-02-20T21:05:19.424 INFO:teuthology.orchestra.run.trial163.stdout:[osd.0] 2026-02-20T21:05:19.424 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCMzJhpcgPZOhAA9dMFHNe5XrKt5wnDH1yXYQ== 2026-02-20T21:05:19.424 INFO:teuthology.orchestra.run.trial163.stdout:[osd.1] 2026-02-20T21:05:19.424 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCNzJhpYxLlIxAA07YSGCF+zto1dq/fTHzJ0g== 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout:[osd.2] 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCOzJhpU0RtCxAATgJomdf6cwjI5MfpjX70sQ== 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout:[client.0] 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCJzJhpzyJZKhAAu/sr0b5F4kLYOK23m3pAzQ== 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror] 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCJzJhpREdmKxAAtwhAwupadmeA3uZ2LduH3A== 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.0] 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCJzJhp0nijMBAA4J5ylIJJ8OoTbttsw7FS8w== 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.1] 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCJzJhpCmPiNRAAFaV+w20LWy/ozhL+MBX9fw== 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.2] 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCJzJhp+TUGOxAADxugmauBY90HnSaq7cbJ3g== 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.3] 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCKzJhpkQSwBBAAw/VJtUfheLNz8GNT7kUU4g== 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.4] 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCKzJhp7e/MCRAA5aLf6SWf934EzY4z6jc/UQ== 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.5] 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCKzJhpqj75DhAAVpB0kRVUfsCBtKXYFKEqRg== 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.6] 2026-02-20T21:05:19.425 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCKzJhpLmAAFBAA3qgMG/Yb+f982GumlUJaNA== 2026-02-20T21:05:19.426 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-20T21:05:19.507 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:05:19.588 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:05:19.667 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-20T21:05:19.746 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:05:19.828 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:05:19.911 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:05:19.992 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:05:20.073 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:05:20.153 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:05:20.234 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:05:20.316 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:05:20.399 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-20T21:05:20.481 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-20T21:05:20.481 DEBUG:teuthology.orchestra.run.trial163:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-20T21:05:20.546 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-20T21:05:20.630 DEBUG:teuthology.orchestra.run.trial163:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-20T21:05:20.694 DEBUG:teuthology.orchestra.run.trial163:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-20T21:05:20.750 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-20T21:05:20.750 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-20T21:05:20.750 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-20T21:05:20.792 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-20T21:05:20.792 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-20T21:05:20.793 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-20T21:05:20.793 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-20T21:05:20.795 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-20T21:05:20.796 DEBUG:tasks.ceph:set 0 configs 2026-02-20T21:05:20.796 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 config dump 2026-02-20T21:05:20.897 INFO:teuthology.orchestra.run.trial163.stdout:WHO MASK LEVEL OPTION VALUE RO 2026-02-20T21:05:20.904 INFO:tasks.ceph:Setting crush tunables to default 2026-02-20T21:05:20.905 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-20T21:05:21.006 INFO:teuthology.orchestra.run.trial163.stderr:adjusted tunables profile to default 2026-02-20T21:05:21.014 INFO:tasks.ceph:check_enable_crimson: False 2026-02-20T21:05:21.015 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-20T21:05:21.015 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:21.015 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-20T21:05:21.039 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:21.039 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-20T21:05:21.103 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:21.103 DEBUG:teuthology.orchestra.run.trial163:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-20T21:05:21.168 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 osd new 956a0026-bd9c-4ea8-b787-0eb4724015b2 0 2026-02-20T21:05:21.313 INFO:teuthology.orchestra.run.trial163.stdout:0 2026-02-20T21:05:21.321 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 osd new fc9a9b6f-73d6-4c05-a019-14d361d3a240 1 2026-02-20T21:05:21.425 INFO:teuthology.orchestra.run.trial163.stdout:1 2026-02-20T21:05:21.433 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 osd new 17e3c46a-bcf7-466e-a56c-33ba58b56170 2 2026-02-20T21:05:21.530 INFO:tasks.ceph.cluster2.mgr.x.trial069.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-20T21:05:21.531 INFO:tasks.ceph.cluster2.mgr.x.trial069.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-20T21:05:21.531 INFO:tasks.ceph.cluster2.mgr.x.trial069.stderr: from numpy import show_config as show_numpy_config 2026-02-20T21:05:21.536 INFO:teuthology.orchestra.run.trial163.stdout:2 2026-02-20T21:05:21.544 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-20T21:05:21.544 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-20T21:05:21.546 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-20T21:05:21.546 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-20T21:05:21.546 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-20T21:05:21.547 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-20T21:05:21.547 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-20T21:05:21.547 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-20T21:05:21.548 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-20T21:05:21.548 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:05:21.648 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:21.648 INFO:teuthology.orchestra.run.trial163.stdout:{"epoch":5,"fsid":"6b7cbdea-dd31-4132-aa0e-90dc10bb8129","created":"2026-02-20T21:05:20.858799+0000","modified":"2026-02-20T21:05:21.536084+0000","last_up_change":"0.000000","last_in_change":"2026-02-20T21:05:21.536084+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"956a0026-bd9c-4ea8-b787-0eb4724015b2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"fc9a9b6f-73d6-4c05-a019-14d361d3a240","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"17e3c46a-bcf7-466e-a56c-33ba58b56170","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:05:21.657 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-20T21:05:21.657 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-20T21:05:21.781 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-20T21:05:21.779+0000 7fdf789b0900 -1 Falling back to public interface 2026-02-20T21:05:21.793 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:05:21.791+0000 7f03da51c900 -1 Falling back to public interface 2026-02-20T21:05:21.802 INFO:tasks.ceph.cluster2.osd.1.trial163.stderr:2026-02-20T21:05:21.800+0000 7fa53d2f7900 -1 Falling back to public interface 2026-02-20T21:05:21.959 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:05:22.222 INFO:teuthology.misc.health.trial163.stderr:2026-02-20T21:05:22.219+0000 7f4fb2ffd640 0 --2- 10.20.193.163:0/4189770842 >> v2:10.20.193.163:3300/0 conn(0x7f4fb4098030 0x7f4fb40ac8d0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2026-02-20T21:05:22.254 INFO:teuthology.misc.health.trial163.stdout: 2026-02-20T21:05:22.254 INFO:teuthology.misc.health.trial163.stdout:{"epoch":5,"fsid":"6b7cbdea-dd31-4132-aa0e-90dc10bb8129","created":"2026-02-20T21:05:20.858799+0000","modified":"2026-02-20T21:05:21.536084+0000","last_up_change":"0.000000","last_in_change":"2026-02-20T21:05:21.536084+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"956a0026-bd9c-4ea8-b787-0eb4724015b2","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"fc9a9b6f-73d6-4c05-a019-14d361d3a240","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"17e3c46a-bcf7-466e-a56c-33ba58b56170","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:05:22.260 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-20T21:05:22.325 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-20T21:05:22.322+0000 7fdf789b0900 -1 osd.2 0 log_to_monitors true 2026-02-20T21:05:22.325 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:05:22.322+0000 7f03da51c900 -1 osd.0 0 log_to_monitors true 2026-02-20T21:05:22.365 INFO:tasks.ceph.cluster2.osd.1.trial163.stderr:2026-02-20T21:05:22.363+0000 7fa53d2f7900 -1 osd.1 0 log_to_monitors true 2026-02-20T21:05:23.555 INFO:tasks.ceph.cluster2.mgr.x.trial069.stderr:2026-02-20T21:05:23.583+0000 7f4272559640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-20T21:05:23.869 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:05:23.867+0000 7f03d5c87640 -1 osd.0 0 waiting for initial osdmap 2026-02-20T21:05:23.880 INFO:tasks.ceph.cluster2.osd.1.trial163.stderr:2026-02-20T21:05:23.879+0000 7fa538a62640 -1 osd.1 0 waiting for initial osdmap 2026-02-20T21:05:23.881 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:05:23.879+0000 7f03cda97640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:05:23.881 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-20T21:05:23.880+0000 7fdf7411d640 -1 osd.2 0 waiting for initial osdmap 2026-02-20T21:05:23.882 INFO:tasks.ceph.cluster2.osd.1.trial163.stderr:2026-02-20T21:05:23.881+0000 7fa530872640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:05:23.883 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-20T21:05:23.881+0000 7fdf6bf2d640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-20T21:05:28.562 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:05:28.734 INFO:teuthology.misc.health.trial163.stdout: 2026-02-20T21:05:28.734 INFO:teuthology.misc.health.trial163.stdout:{"epoch":11,"fsid":"6b7cbdea-dd31-4132-aa0e-90dc10bb8129","created":"2026-02-20T21:05:20.858799+0000","modified":"2026-02-20T21:05:27.871633+0000","last_up_change":"2026-02-20T21:05:24.866038+0000","last_in_change":"2026-02-20T21:05:21.536084+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:05:26.567416+0000","flags":32769,"flags_names":"hashpspool,creating","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"956a0026-bd9c-4ea8-b787-0eb4724015b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6808","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6809","nonce":2921121942}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6810","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6811","nonce":2921121942}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6814","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6815","nonce":2921121942}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6812","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6813","nonce":2921121942}]},"public_addr":"10.20.193.163:6809/2921121942","cluster_addr":"10.20.193.163:6811/2921121942","heartbeat_back_addr":"10.20.193.163:6815/2921121942","heartbeat_front_addr":"10.20.193.163:6813/2921121942","state":["exists","up"]},{"osd":1,"uuid":"fc9a9b6f-73d6-4c05-a019-14d361d3a240","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6816","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6817","nonce":3447493578}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6818","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6819","nonce":3447493578}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6822","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6823","nonce":3447493578}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6820","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6821","nonce":3447493578}]},"public_addr":"10.20.193.163:6817/3447493578","cluster_addr":"10.20.193.163:6819/3447493578","heartbeat_back_addr":"10.20.193.163:6823/3447493578","heartbeat_front_addr":"10.20.193.163:6821/3447493578","state":["exists","up"]},{"osd":2,"uuid":"17e3c46a-bcf7-466e-a56c-33ba58b56170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6800","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6801","nonce":3294996969}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6802","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6803","nonce":3294996969}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6806","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6807","nonce":3294996969}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6804","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6805","nonce":3294996969}]},"public_addr":"10.20.193.163:6801/3294996969","cluster_addr":"10.20.193.163:6803/3294996969","heartbeat_back_addr":"10.20.193.163:6807/3294996969","heartbeat_front_addr":"10.20.193.163:6805/3294996969","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:23.323693+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:23.393980+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:23.304166+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:05:28.741 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-20T21:05:28.741 INFO:tasks.ceph:Creating RBD pool 2026-02-20T21:05:28.741 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-20T21:05:29.877 INFO:teuthology.orchestra.run.trial163.stderr:pool 'rbd' created 2026-02-20T21:05:29.890 DEBUG:teuthology.orchestra.run.trial163:> rbd --cluster cluster2 pool init rbd 2026-02-20T21:05:32.889 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-20T21:05:32.890 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 config log 1 --format=json 2026-02-20T21:05:32.891 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-20T21:05:33.071 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:33.080 INFO:teuthology.orchestra.run.trial163.stdout:[{"version":1,"timestamp":"0.000000","name":"","changes":[]}] 2026-02-20T21:05:33.080 INFO:tasks.ceph_manager:config epoch is 1 2026-02-20T21:05:33.081 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-20T21:05:33.081 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-20T21:05:33.081 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-20T21:05:33.296 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:33.305 INFO:teuthology.orchestra.run.trial163.stdout:{"epoch":5,"flags":0,"active_gid":4101,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6826","nonce":3694319539},{"type":"v1","addr":"10.20.193.69:6827","nonce":3694319539}]},"active_addr":"10.20.193.69:6827/3694319539","active_change":"2026-02-20T21:05:22.546599+0000","active_mgr_features":4544132024016699391,"available":true,"standbys":[],"modules":["iostat","nfs"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"certificate_automated_rotation_enabled":{"name":"certificate_automated_rotation_enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"This flag controls whether cephadm automatically rotates certificates upon expiration.","long_desc":"","tags":[],"see_also":[]},"certificate_check_debug_mode":{"name":"certificate_check_debug_mode","type":"bool","level":"dev","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"FOR TESTING ONLY: This flag forces the certificate check instead of waiting for certificate_check_period.","long_desc":"","tags":[],"see_also":[]},"certificate_check_period":{"name":"certificate_check_period","type":"int","level":"advanced","flags":0,"default_value":"1","min":"0","max":"30","enum_allowed":[],"desc":"Specifies how often (in days) the certificate should be checked for validity.","long_desc":"","tags":[],"see_also":[]},"certificate_duration_days":{"name":"certificate_duration_days","type":"int","level":"advanced","flags":0,"default_value":"1095","min":"90","max":"3650","enum_allowed":[],"desc":"Specifies the duration of self certificates generated and signed by cephadm root CA","long_desc":"","tags":[],"see_also":[]},"certificate_renewal_threshold_days":{"name":"certificate_renewal_threshold_days","type":"int","level":"advanced","flags":0,"default_value":"30","min":"10","max":"90","enum_allowed":[],"desc":"Specifies the lead time in days to initiate certificate renewal before expiration.","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.28.1","min":"","max":"","enum_allowed":[],"desc":"Alertmanager container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"Elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:12.3.1","min":"","max":"","enum_allowed":[],"desc":"Grafana container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"Haproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.9.1","min":"","max":"","enum_allowed":[],"desc":"Node exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.5","min":"","max":"","enum_allowed":[],"desc":"Nvmeof container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"Oauth2 proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v3.6.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:ceph20-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba metrics container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"Snmp gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"stray_daemon_check_interval":{"name":"stray_daemon_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"how frequently cephadm should check for the presence of stray daemons","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROM_ALERT_CREDENTIAL_CACHE_TTL":{"name":"PROM_ALERT_CREDENTIAL_CACHE_TTL","type":"int","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_HOSTNAME_PER_DAEMON":{"name":"RGW_HOSTNAME_PER_DAEMON","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crypto_caller":{"name":"crypto_caller","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sso_oauth2":{"name":"sso_oauth2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"prometheus_tls_secret_name":{"name":"prometheus_tls_secret_name","type":"str","level":"advanced","flags":0,"default_value":"rook-ceph-prometheus-server-tls","min":"","max":"","enum_allowed":[],"desc":"name of tls secret in k8s for prometheus","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_cloning":{"name":"pause_cloning","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"pause_purging":{"name":"pause_purging","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Pause asynchronous subvolume purge threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"tentacle":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.69:0","nonce":3399291256}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.69:0","nonce":2625037897}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.69:0","nonce":3637335482}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.69:0","nonce":4046148587}]}]} 2026-02-20T21:05:33.316 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-20T21:05:33.316 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-20T21:05:33.316 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:05:33.496 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:33.496 INFO:teuthology.orchestra.run.trial163.stdout:{"epoch":16,"fsid":"6b7cbdea-dd31-4132-aa0e-90dc10bb8129","created":"2026-02-20T21:05:20.858799+0000","modified":"2026-02-20T21:05:32.881184+0000","last_up_change":"2026-02-20T21:05:24.866038+0000","last_in_change":"2026-02-20T21:05:21.536084+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:05:26.567416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:05:28.922053+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"956a0026-bd9c-4ea8-b787-0eb4724015b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6808","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6809","nonce":2921121942}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6810","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6811","nonce":2921121942}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6814","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6815","nonce":2921121942}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6812","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6813","nonce":2921121942}]},"public_addr":"10.20.193.163:6809/2921121942","cluster_addr":"10.20.193.163:6811/2921121942","heartbeat_back_addr":"10.20.193.163:6815/2921121942","heartbeat_front_addr":"10.20.193.163:6813/2921121942","state":["exists","up"]},{"osd":1,"uuid":"fc9a9b6f-73d6-4c05-a019-14d361d3a240","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6816","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6817","nonce":3447493578}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6818","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6819","nonce":3447493578}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6822","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6823","nonce":3447493578}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6820","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6821","nonce":3447493578}]},"public_addr":"10.20.193.163:6817/3447493578","cluster_addr":"10.20.193.163:6819/3447493578","heartbeat_back_addr":"10.20.193.163:6823/3447493578","heartbeat_front_addr":"10.20.193.163:6821/3447493578","state":["exists","up"]},{"osd":2,"uuid":"17e3c46a-bcf7-466e-a56c-33ba58b56170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6800","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6801","nonce":3294996969}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6802","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6803","nonce":3294996969}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6806","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6807","nonce":3294996969}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6804","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6805","nonce":3294996969}]},"public_addr":"10.20.193.163:6801/3294996969","cluster_addr":"10.20.193.163:6803/3294996969","heartbeat_back_addr":"10.20.193.163:6807/3294996969","heartbeat_front_addr":"10.20.193.163:6805/3294996969","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:23.323693+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:23.393980+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:23.304166+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:05:33.505 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-20T21:05:33.505 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:05:33.688 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:33.688 INFO:teuthology.orchestra.run.trial163.stdout:{"epoch":16,"fsid":"6b7cbdea-dd31-4132-aa0e-90dc10bb8129","created":"2026-02-20T21:05:20.858799+0000","modified":"2026-02-20T21:05:32.881184+0000","last_up_change":"2026-02-20T21:05:24.866038+0000","last_in_change":"2026-02-20T21:05:21.536084+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:05:26.567416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:05:28.922053+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"956a0026-bd9c-4ea8-b787-0eb4724015b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6808","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6809","nonce":2921121942}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6810","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6811","nonce":2921121942}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6814","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6815","nonce":2921121942}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6812","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6813","nonce":2921121942}]},"public_addr":"10.20.193.163:6809/2921121942","cluster_addr":"10.20.193.163:6811/2921121942","heartbeat_back_addr":"10.20.193.163:6815/2921121942","heartbeat_front_addr":"10.20.193.163:6813/2921121942","state":["exists","up"]},{"osd":1,"uuid":"fc9a9b6f-73d6-4c05-a019-14d361d3a240","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6816","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6817","nonce":3447493578}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6818","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6819","nonce":3447493578}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6822","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6823","nonce":3447493578}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6820","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6821","nonce":3447493578}]},"public_addr":"10.20.193.163:6817/3447493578","cluster_addr":"10.20.193.163:6819/3447493578","heartbeat_back_addr":"10.20.193.163:6823/3447493578","heartbeat_front_addr":"10.20.193.163:6821/3447493578","state":["exists","up"]},{"osd":2,"uuid":"17e3c46a-bcf7-466e-a56c-33ba58b56170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6800","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6801","nonce":3294996969}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6802","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6803","nonce":3294996969}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6806","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6807","nonce":3294996969}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6804","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6805","nonce":3294996969}]},"public_addr":"10.20.193.163:6801/3294996969","cluster_addr":"10.20.193.163:6803/3294996969","heartbeat_back_addr":"10.20.193.163:6807/3294996969","heartbeat_front_addr":"10.20.193.163:6805/3294996969","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:23.323693+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:23.393980+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:23.304166+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:05:33.698 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-20T21:05:33.698 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-20T21:05:33.699 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-20T21:05:33.788 INFO:teuthology.orchestra.run.trial163.stdout:34359738372 2026-02-20T21:05:33.789 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-20T21:05:33.790 INFO:teuthology.orchestra.run.trial163.stdout:34359738373 2026-02-20T21:05:33.790 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-20T21:05:33.791 INFO:teuthology.orchestra.run.trial163.stdout:34359738371 2026-02-20T21:05:33.792 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-20T21:05:34.012 INFO:teuthology.orchestra.run.trial163.stdout:34359738372 2026-02-20T21:05:34.013 INFO:teuthology.orchestra.run.trial163.stdout:34359738371 2026-02-20T21:05:34.014 INFO:teuthology.orchestra.run.trial163.stdout:34359738370 2026-02-20T21:05:34.024 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738372 for osd.1 2026-02-20T21:05:34.025 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-02-20T21:05:34.025 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.2 2026-02-20T21:05:35.024 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-20T21:05:35.025 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-20T21:05:35.026 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-20T21:05:35.207 INFO:teuthology.orchestra.run.trial163.stdout:34359738373 2026-02-20T21:05:35.212 INFO:teuthology.orchestra.run.trial163.stdout:34359738371 2026-02-20T21:05:35.213 INFO:teuthology.orchestra.run.trial163.stdout:34359738372 2026-02-20T21:05:35.217 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738373 for osd.1 2026-02-20T21:05:35.217 DEBUG:teuthology.parallel:result is None 2026-02-20T21:05:35.223 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738371 for osd.2 2026-02-20T21:05:35.224 DEBUG:teuthology.parallel:result is None 2026-02-20T21:05:35.224 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738372 for osd.0 2026-02-20T21:05:35.224 DEBUG:teuthology.parallel:result is None 2026-02-20T21:05:35.224 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-20T21:05:35.224 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:05:35.441 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:35.442 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-20T21:05:35.449 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2026-02-20T21:05:34.582542+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82456,"kb_used_data":1576,"kb_used_omap":24,"kb_used_meta":80807,"kb_avail":283033064,"statfs":{"total":289910292480,"available":289825857536,"internally_reserved":0,"allocated":1613824,"data_stored":1303003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25024,"internal_metadata":82746944},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.672300"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.885321+0000","last_change":"2026-02-20T21:05:32.885706+0000","last_active":"2026-02-20T21:05:32.885321+0000","last_peered":"2026-02-20T21:05:32.885321+0000","last_clean":"2026-02-20T21:05:32.885321+0000","last_became_active":"2026-02-20T21:05:30.883104+0000","last_became_peered":"2026-02-20T21:05:30.883104+0000","last_unstale":"2026-02-20T21:05:32.885321+0000","last_undegraded":"2026-02-20T21:05:32.885321+0000","last_fullsized":"2026-02-20T21:05:32.885321+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:14:32.329895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010312629999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.885114+0000","last_change":"2026-02-20T21:05:32.885327+0000","last_active":"2026-02-20T21:05:32.885114+0000","last_peered":"2026-02-20T21:05:32.885114+0000","last_clean":"2026-02-20T21:05:32.885114+0000","last_became_active":"2026-02-20T21:05:30.882952+0000","last_became_peered":"2026-02-20T21:05:30.882952+0000","last_unstale":"2026-02-20T21:05:32.885114+0000","last_undegraded":"2026-02-20T21:05:32.885114+0000","last_fullsized":"2026-02-20T21:05:32.885114+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:32:38.163582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071972500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.884896+0000","last_change":"2026-02-20T21:05:32.885072+0000","last_active":"2026-02-20T21:05:32.884896+0000","last_peered":"2026-02-20T21:05:32.884896+0000","last_clean":"2026-02-20T21:05:32.884896+0000","last_became_active":"2026-02-20T21:05:30.883044+0000","last_became_peered":"2026-02-20T21:05:30.883044+0000","last_unstale":"2026-02-20T21:05:32.884896+0000","last_undegraded":"2026-02-20T21:05:32.884896+0000","last_fullsized":"2026-02-20T21:05:32.884896+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:57:08.905691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055306500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.884855+0000","last_change":"2026-02-20T21:05:32.885039+0000","last_active":"2026-02-20T21:05:32.884855+0000","last_peered":"2026-02-20T21:05:32.884855+0000","last_clean":"2026-02-20T21:05:32.884855+0000","last_became_active":"2026-02-20T21:05:30.883115+0000","last_became_peered":"2026-02-20T21:05:30.883115+0000","last_unstale":"2026-02-20T21:05:32.884855+0000","last_undegraded":"2026-02-20T21:05:32.884855+0000","last_fullsized":"2026-02-20T21:05:32.884855+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:11:05.945358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052496400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":22,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.884158+0000","last_change":"2026-02-20T21:05:32.883927+0000","last_active":"2026-02-20T21:05:32.884158+0000","last_peered":"2026-02-20T21:05:32.884158+0000","last_clean":"2026-02-20T21:05:32.884158+0000","last_became_active":"2026-02-20T21:05:30.883032+0000","last_became_peered":"2026-02-20T21:05:30.883032+0000","last_unstale":"2026-02-20T21:05:32.884158+0000","last_undegraded":"2026-02-20T21:05:32.884158+0000","last_fullsized":"2026-02-20T21:05:32.884158+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:07:25.195375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077015799999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.926973+0000","last_change":"2026-02-20T21:05:32.927062+0000","last_active":"2026-02-20T21:05:32.926973+0000","last_peered":"2026-02-20T21:05:32.926973+0000","last_clean":"2026-02-20T21:05:32.926973+0000","last_became_active":"2026-02-20T21:05:30.883679+0000","last_became_peered":"2026-02-20T21:05:30.883679+0000","last_unstale":"2026-02-20T21:05:32.926973+0000","last_undegraded":"2026-02-20T21:05:32.926973+0000","last_fullsized":"2026-02-20T21:05:32.926973+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:43:48.880973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021062000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.926994+0000","last_change":"2026-02-20T21:05:32.927079+0000","last_active":"2026-02-20T21:05:32.926994+0000","last_peered":"2026-02-20T21:05:32.926994+0000","last_clean":"2026-02-20T21:05:32.926994+0000","last_became_active":"2026-02-20T21:05:30.883572+0000","last_became_peered":"2026-02-20T21:05:30.883572+0000","last_unstale":"2026-02-20T21:05:32.926994+0000","last_undegraded":"2026-02-20T21:05:32.926994+0000","last_fullsized":"2026-02-20T21:05:32.926994+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:29:24.098186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022568800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.885069+0000","last_change":"2026-02-20T21:05:32.885282+0000","last_active":"2026-02-20T21:05:32.885069+0000","last_peered":"2026-02-20T21:05:32.885069+0000","last_clean":"2026-02-20T21:05:32.885069+0000","last_became_active":"2026-02-20T21:05:30.882188+0000","last_became_peered":"2026-02-20T21:05:30.882188+0000","last_unstale":"2026-02-20T21:05:32.885069+0000","last_undegraded":"2026-02-20T21:05:32.885069+0000","last_fullsized":"2026-02-20T21:05:32.885069+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:59:42.030633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075006200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":111,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.884304+0000","last_change":"2026-02-20T21:05:28.168843+0000","last_active":"2026-02-20T21:05:32.884304+0000","last_peered":"2026-02-20T21:05:32.884304+0000","last_clean":"2026-02-20T21:05:32.884304+0000","last_became_active":"2026-02-20T21:05:28.168224+0000","last_became_peered":"2026-02-20T21:05:28.168224+0000","last_unstale":"2026-02-20T21:05:32.884304+0000","last_undegraded":"2026-02-20T21:05:32.884304+0000","last_fullsized":"2026-02-20T21:05:32.884304+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:26.870888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:26.870888+0000","last_clean_scrub_stamp":"2026-02-20T21:05:26.870888+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:36:01.610328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27096,"kb_used_data":136,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344744,"statfs":{"total":96636764160,"available":96609017856,"internally_reserved":0,"allocated":139264,"data_stored":40743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7474,"internal_metadata":27583182},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27680,"kb_used_data":720,"kb_used_omap":8,"kb_used_meta":26935,"kb_avail":94344160,"statfs":{"total":96636764160,"available":96608419840,"internally_reserved":0,"allocated":737280,"data_stored":631130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8775,"internal_metadata":27581881},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27680,"kb_used_data":720,"kb_used_omap":8,"kb_used_meta":26935,"kb_avail":94344160,"statfs":{"total":96636764160,"available":96608419840,"internally_reserved":0,"allocated":737280,"data_stored":631130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8775,"internal_metadata":27581881},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:05:35.451 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:05:35.629 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:35.629 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-20T21:05:35.637 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2026-02-20T21:05:34.582542+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82456,"kb_used_data":1576,"kb_used_omap":24,"kb_used_meta":80807,"kb_avail":283033064,"statfs":{"total":289910292480,"available":289825857536,"internally_reserved":0,"allocated":1613824,"data_stored":1303003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25024,"internal_metadata":82746944},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.672300"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.885321+0000","last_change":"2026-02-20T21:05:32.885706+0000","last_active":"2026-02-20T21:05:32.885321+0000","last_peered":"2026-02-20T21:05:32.885321+0000","last_clean":"2026-02-20T21:05:32.885321+0000","last_became_active":"2026-02-20T21:05:30.883104+0000","last_became_peered":"2026-02-20T21:05:30.883104+0000","last_unstale":"2026-02-20T21:05:32.885321+0000","last_undegraded":"2026-02-20T21:05:32.885321+0000","last_fullsized":"2026-02-20T21:05:32.885321+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:14:32.329895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010312629999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.885114+0000","last_change":"2026-02-20T21:05:32.885327+0000","last_active":"2026-02-20T21:05:32.885114+0000","last_peered":"2026-02-20T21:05:32.885114+0000","last_clean":"2026-02-20T21:05:32.885114+0000","last_became_active":"2026-02-20T21:05:30.882952+0000","last_became_peered":"2026-02-20T21:05:30.882952+0000","last_unstale":"2026-02-20T21:05:32.885114+0000","last_undegraded":"2026-02-20T21:05:32.885114+0000","last_fullsized":"2026-02-20T21:05:32.885114+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:32:38.163582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071972500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.884896+0000","last_change":"2026-02-20T21:05:32.885072+0000","last_active":"2026-02-20T21:05:32.884896+0000","last_peered":"2026-02-20T21:05:32.884896+0000","last_clean":"2026-02-20T21:05:32.884896+0000","last_became_active":"2026-02-20T21:05:30.883044+0000","last_became_peered":"2026-02-20T21:05:30.883044+0000","last_unstale":"2026-02-20T21:05:32.884896+0000","last_undegraded":"2026-02-20T21:05:32.884896+0000","last_fullsized":"2026-02-20T21:05:32.884896+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:57:08.905691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055306500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.884855+0000","last_change":"2026-02-20T21:05:32.885039+0000","last_active":"2026-02-20T21:05:32.884855+0000","last_peered":"2026-02-20T21:05:32.884855+0000","last_clean":"2026-02-20T21:05:32.884855+0000","last_became_active":"2026-02-20T21:05:30.883115+0000","last_became_peered":"2026-02-20T21:05:30.883115+0000","last_unstale":"2026-02-20T21:05:32.884855+0000","last_undegraded":"2026-02-20T21:05:32.884855+0000","last_fullsized":"2026-02-20T21:05:32.884855+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:11:05.945358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052496400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":22,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.884158+0000","last_change":"2026-02-20T21:05:32.883927+0000","last_active":"2026-02-20T21:05:32.884158+0000","last_peered":"2026-02-20T21:05:32.884158+0000","last_clean":"2026-02-20T21:05:32.884158+0000","last_became_active":"2026-02-20T21:05:30.883032+0000","last_became_peered":"2026-02-20T21:05:30.883032+0000","last_unstale":"2026-02-20T21:05:32.884158+0000","last_undegraded":"2026-02-20T21:05:32.884158+0000","last_fullsized":"2026-02-20T21:05:32.884158+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:07:25.195375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077015799999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.926973+0000","last_change":"2026-02-20T21:05:32.927062+0000","last_active":"2026-02-20T21:05:32.926973+0000","last_peered":"2026-02-20T21:05:32.926973+0000","last_clean":"2026-02-20T21:05:32.926973+0000","last_became_active":"2026-02-20T21:05:30.883679+0000","last_became_peered":"2026-02-20T21:05:30.883679+0000","last_unstale":"2026-02-20T21:05:32.926973+0000","last_undegraded":"2026-02-20T21:05:32.926973+0000","last_fullsized":"2026-02-20T21:05:32.926973+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:43:48.880973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021062000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.926994+0000","last_change":"2026-02-20T21:05:32.927079+0000","last_active":"2026-02-20T21:05:32.926994+0000","last_peered":"2026-02-20T21:05:32.926994+0000","last_clean":"2026-02-20T21:05:32.926994+0000","last_became_active":"2026-02-20T21:05:30.883572+0000","last_became_peered":"2026-02-20T21:05:30.883572+0000","last_unstale":"2026-02-20T21:05:32.926994+0000","last_undegraded":"2026-02-20T21:05:32.926994+0000","last_fullsized":"2026-02-20T21:05:32.926994+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:29:24.098186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022568800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.885069+0000","last_change":"2026-02-20T21:05:32.885282+0000","last_active":"2026-02-20T21:05:32.885069+0000","last_peered":"2026-02-20T21:05:32.885069+0000","last_clean":"2026-02-20T21:05:32.885069+0000","last_became_active":"2026-02-20T21:05:30.882188+0000","last_became_peered":"2026-02-20T21:05:30.882188+0000","last_unstale":"2026-02-20T21:05:32.885069+0000","last_undegraded":"2026-02-20T21:05:32.885069+0000","last_fullsized":"2026-02-20T21:05:32.885069+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:59:42.030633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075006200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":111,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-20T21:05:32.884304+0000","last_change":"2026-02-20T21:05:28.168843+0000","last_active":"2026-02-20T21:05:32.884304+0000","last_peered":"2026-02-20T21:05:32.884304+0000","last_clean":"2026-02-20T21:05:32.884304+0000","last_became_active":"2026-02-20T21:05:28.168224+0000","last_became_peered":"2026-02-20T21:05:28.168224+0000","last_unstale":"2026-02-20T21:05:32.884304+0000","last_undegraded":"2026-02-20T21:05:32.884304+0000","last_fullsized":"2026-02-20T21:05:32.884304+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:26.870888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:26.870888+0000","last_clean_scrub_stamp":"2026-02-20T21:05:26.870888+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:36:01.610328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738371,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27096,"kb_used_data":136,"kb_used_omap":7,"kb_used_meta":26936,"kb_avail":94344744,"statfs":{"total":96636764160,"available":96609017856,"internally_reserved":0,"allocated":139264,"data_stored":40743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7474,"internal_metadata":27583182},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27680,"kb_used_data":720,"kb_used_omap":8,"kb_used_meta":26935,"kb_avail":94344160,"statfs":{"total":96636764160,"available":96608419840,"internally_reserved":0,"allocated":737280,"data_stored":631130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8775,"internal_metadata":27581881},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27680,"kb_used_data":720,"kb_used_omap":8,"kb_used_meta":26935,"kb_avail":94344160,"statfs":{"total":96636764160,"available":96608419840,"internally_reserved":0,"allocated":737280,"data_stored":631130,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8775,"internal_metadata":27581881},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:05:35.639 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-20T21:05:35.639 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-20T21:05:35.639 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-20T21:05:35.639 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-20T21:05:35.845 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:05:35.845 INFO:teuthology.orchestra.run.trial163.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-20T21:05:35.854 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-20T21:05:35.854 INFO:teuthology.run_tasks:Running task exec... 2026-02-20T21:05:35.869 INFO:teuthology.task.exec:Executing custom commands... 2026-02-20T21:05:35.870 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial163.front.sepia.ceph.com 2026-02-20T21:05:35.870 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:05:36.101 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror] 2026-02-20T21:05:36.101 INFO:teuthology.orchestra.run.trial163.stdout: key = AQBzzJhp5QUjJhAASSPijdDFaZGDomWwbZMQ8Q== 2026-02-20T21:05:36.101 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-20T21:05:36.101 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-20T21:05:36.101 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror 2026-02-20T21:05:36.111 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:05:36.340 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.0] 2026-02-20T21:05:36.340 INFO:teuthology.orchestra.run.trial163.stdout: key = AQBzzJhpQMNJKxAApZdPirQ1TNbz3+lAV3EHGg== 2026-02-20T21:05:36.341 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:05:36.341 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-20T21:05:36.341 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.0 2026-02-20T21:05:36.351 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:05:36.578 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.1] 2026-02-20T21:05:36.578 INFO:teuthology.orchestra.run.trial163.stdout: key = AQBzzJhpYW+HMBAASMQhrR1r7dMCWM4Ihx3yRg== 2026-02-20T21:05:36.578 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:05:36.578 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-20T21:05:36.578 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.1 2026-02-20T21:05:36.588 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:05:36.816 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.2] 2026-02-20T21:05:36.816 INFO:teuthology.orchestra.run.trial163.stdout: key = AQBzzJhpi/qqNRAA50IdPKbE0dVm3KNcCWMBQA== 2026-02-20T21:05:36.816 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:05:36.817 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-20T21:05:36.817 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.2 2026-02-20T21:05:36.825 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:05:37.054 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.3] 2026-02-20T21:05:37.054 INFO:teuthology.orchestra.run.trial163.stdout: key = AQBzzJhpBECuOhAA570yrT9FfMr6to2tYoZpaA== 2026-02-20T21:05:37.054 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:05:37.054 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-20T21:05:37.055 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.3 2026-02-20T21:05:37.064 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:05:37.293 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror] 2026-02-20T21:05:37.293 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCJzJhpREdmKxAAtwhAwupadmeA3uZ2LduH3A== 2026-02-20T21:05:37.293 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-20T21:05:37.293 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-20T21:05:37.293 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror 2026-02-20T21:05:37.304 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:05:37.532 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.0] 2026-02-20T21:05:37.532 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCJzJhp0nijMBAA4J5ylIJJ8OoTbttsw7FS8w== 2026-02-20T21:05:37.532 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:05:37.533 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-20T21:05:37.533 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.0 2026-02-20T21:05:37.543 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:05:37.776 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.1] 2026-02-20T21:05:37.776 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCJzJhpCmPiNRAAFaV+w20LWy/ozhL+MBX9fw== 2026-02-20T21:05:37.776 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:05:37.776 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-20T21:05:37.777 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.1 2026-02-20T21:05:37.787 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:05:38.017 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.2] 2026-02-20T21:05:38.017 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCJzJhp+TUGOxAADxugmauBY90HnSaq7cbJ3g== 2026-02-20T21:05:38.017 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:05:38.018 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-20T21:05:38.018 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.2 2026-02-20T21:05:38.028 DEBUG:teuthology.orchestra.run.trial163:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-20T21:05:38.256 INFO:teuthology.orchestra.run.trial163.stdout:[client.mirror.3] 2026-02-20T21:05:38.256 INFO:teuthology.orchestra.run.trial163.stdout: key = AQCKzJhpkQSwBBAAw/VJtUfheLNz8GNT7kUU4g== 2026-02-20T21:05:38.256 INFO:teuthology.orchestra.run.trial163.stdout: caps mon = "profile rbd-mirror" 2026-02-20T21:05:38.256 INFO:teuthology.orchestra.run.trial163.stdout: caps osd = "profile rbd" 2026-02-20T21:05:38.257 INFO:teuthology.orchestra.run.trial163.stderr:updated caps for client.mirror.3 2026-02-20T21:05:38.266 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:05:38.286 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-20T21:05:38.286 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-20T21:05:38.289 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-20T21:05:38.289 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:05:38.304 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-20T21:05:38.304 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-20T21:05:38.307 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-20T21:05:38.307 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:05:38.321 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-20T21:05:38.321 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-20T21:05:38.324 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-20T21:05:38.324 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:05:38.341 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-20T21:05:38.342 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-20T21:05:38.344 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-20T21:05:38.344 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:05:38.359 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T21:05:38.360 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T21:05:38.362 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T21:05:38.362 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:05:38.416 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-20T21:05:38.416 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-20T21:05:38.418 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-20T21:05:38.418 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:05:38.428 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-20T21:05:38.428 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-20T21:05:38.430 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-20T21:05:38.430 INFO:teuthology.run_tasks:Running task rbd-mirror... 2026-02-20T21:05:38.442 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-20T21:05:38.442 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-20T21:05:38.444 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-20T21:05:38.444 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-02-20T21:05:38.459 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1771621538 2026-02-20T21:05:38.459 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-02-20T21:05:38.459 INFO:teuthology.run_tasks:Running task rbd-mirror-thrash... 2026-02-20T21:05:38.472 INFO:tasks.rbd_mirror_thrash:rbd_mirror_thrash using random seed: 1771621538 2026-02-20T21:05:38.472 DEBUG:tasks.rbd_mirror_thrash:Yielding 2026-02-20T21:05:38.472 INFO:teuthology.run_tasks:Running task workunit... 2026-02-20T21:05:38.493 INFO:tasks.workunit:Pulling workunits from ref f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:05:38.493 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-20T21:05:38.494 DEBUG:teuthology.orchestra.run.trial163:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-20T21:05:38.494 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster1 2026-02-20T21:05:38.494 INFO:tasks.rbd_mirror_thrash:waiting for 94.01475809225325 secs before thrashing 2026-02-20T21:05:38.494 INFO:tasks.rbd_mirror_thrash:starting thrash for cluster cluster2 2026-02-20T21:05:38.494 INFO:tasks.rbd_mirror_thrash:waiting for 87.55854887723795 secs before thrashing 2026-02-20T21:05:38.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:05:38.510 INFO:teuthology.orchestra.run.trial163.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-20T21:05:38.511 DEBUG:teuthology.orchestra.run.trial163:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-20T21:05:38.567 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-20T21:05:38.567 DEBUG:teuthology.orchestra.run.trial163:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-20T21:05:38.623 INFO:tasks.workunit:timeout=6h 2026-02-20T21:05:38.623 INFO:tasks.workunit:cleanup=True 2026-02-20T21:05:38.624 DEBUG:teuthology.orchestra.run.trial163:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout f7870454149f22574086dd2ff55d050585136531 2026-02-20T21:05:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-20T21:05:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Note: switching to 'f7870454149f22574086dd2ff55d050585136531'. 2026-02-20T21:05:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-20T21:05:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-20T21:05:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-20T21:05:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:state without impacting any branches by switching back to a branch. 2026-02-20T21:05:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-20T21:05:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-20T21:05:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-20T21:05:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-20T21:05:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: git switch -c 2026-02-20T21:05:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-20T21:05:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Or undo this operation with: 2026-02-20T21:05:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-20T21:05:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: git switch - 2026-02-20T21:05:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-20T21:05:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-20T21:05:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: 2026-02-20T21:05:58.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:HEAD is now at f7870454149 ceph-dev-pipeline: configure 2026-02-20T21:05:58.544 DEBUG:teuthology.orchestra.run.trial163:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-20T21:05:58.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-20T21:05:58.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-20T21:05:58.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-20T21:05:58.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-20T21:05:58.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-20T21:05:58.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-20T21:05:58.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-20T21:05:58.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-20T21:05:58.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-20T21:05:58.785 DEBUG:teuthology.orchestra.run.trial163:> set -ex 2026-02-20T21:05:58.785 DEBUG:teuthology.orchestra.run.trial163:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-20T21:05:58.843 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_stress.sh on cluster1.client.mirror... 2026-02-20T21:05:58.845 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_stress.sh... 2026-02-20T21:05:58.846 DEBUG:teuthology.orchestra.run.trial163:workunit test rbd/rbd_mirror_stress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=f7870454149f22574086dd2ff55d050585136531 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=pool RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=journal RBD_MIRROR_USE_EXISTING_CLUSTER=1 RBD_MIRROR_USE_RBD_MIRROR=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-20T21:05:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ IMAGE_COUNT=50 2026-02-20T21:05:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ export LOCKDEP=0 2026-02-20T21:05:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ LOCKDEP=0 2026-02-20T21:05:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-20T21:05:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-20T21:05:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ RBD_MIRROR_INSTANCES=4 2026-02-20T21:05:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CLUSTER1=cluster1 2026-02-20T21:05:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CLUSTER2=cluster2 2026-02-20T21:05:58.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-20T21:05:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ POOL=mirror 2026-02-20T21:05:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ PARENT_POOL=mirror_parent 2026-02-20T21:05:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ NS1=ns1 2026-02-20T21:05:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ NS2=ns2 2026-02-20T21:05:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ TEMPDIR= 2026-02-20T21:05:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ID=mirror 2026-02-20T21:05:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,journaling 2026-02-20T21:05:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-20T21:05:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ RBD_MIRROR_MODE=journal 2026-02-20T21:05:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ MIRROR_POOL_MODE=pool 2026-02-20T21:05:58.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' journal = snapshot ']' 2026-02-20T21:05:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-20T21:05:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS='--id mirror' 2026-02-20T21:05:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ LAST_MIRROR_INSTANCE=3 2026-02-20T21:05:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_stress.sh 2026-02-20T21:05:58.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-20T21:05:58.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-20T21:05:58.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_BIN=. 2026-02-20T21:05:58.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_SRC=. 2026-02-20T21:05:58.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-20T21:05:58.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 -gt 0 ']' 2026-02-20T21:05:58.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ setup 2026-02-20T21:05:58.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local c 2026-02-20T21:05:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-20T21:05:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ setup_tempdir 2026-02-20T21:05:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:05:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ mktemp -d 2026-02-20T21:05:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ TEMPDIR=/tmp/tmp.IQNOxgiLiE 2026-02-20T21:05:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z 1 ']' 2026-02-20T21:05:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ setup_pools cluster1 cluster2 2026-02-20T21:05:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:05:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_cluster=cluster2 2026-02-20T21:05:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mon_map_file 2026-02-20T21:05:58.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mon_addr 2026-02-20T21:05:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local admin_key_file 2026-02-20T21:05:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local uuid 2026-02-20T21:05:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:05:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-20T21:05:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror' already exists 2026-02-20T21:05:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:05:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-20T21:06:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror_parent' already exists 2026-02-20T21:06:00.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:06:00.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-20T21:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:06:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-20T21:06:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:06:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:06:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror pool 2026-02-20T21:06:06.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-20T21:06:06.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-20T21:06:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-20T21:06:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-20T21:06:07.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 pool 2026-02-20T21:06:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-20T21:06:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 pool 2026-02-20T21:06:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z ']' 2026-02-20T21:06:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z ']' 2026-02-20T21:06:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_add cluster1 mirror cluster2 2026-02-20T21:06:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:06:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local client_cluster=cluster2 2026-02-20T21:06:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_cluster=cluster2 2026-02-20T21:06:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local uuid_var_name 2026-02-20T21:06:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:06:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local error_code 2026-02-20T21:06:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local peer_uuid 2026-02-20T21:06:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 16 32 2026-02-20T21:06:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:06:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2026-02-20T21:06:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_uuid=9d54e664-d69d-4c02-9066-950093e2d260 2026-02-20T21:06:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-20T21:06:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:06:07.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 17 ']' 2026-02-20T21:06:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 -eq 0 2026-02-20T21:06:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:06:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:06:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_add cluster1 mirror_parent cluster2 2026-02-20T21:06:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:06:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror_parent 2026-02-20T21:06:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local client_cluster=cluster2 2026-02-20T21:06:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_cluster=cluster2 2026-02-20T21:06:07.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local uuid_var_name 2026-02-20T21:06:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:06:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local error_code 2026-02-20T21:06:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local peer_uuid 2026-02-20T21:06:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 16 32 2026-02-20T21:06:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:06:07.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2026-02-20T21:06:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_uuid=8e917922-9fe4-45e5-83d3-25635852754c 2026-02-20T21:06:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-20T21:06:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:06:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 17 ']' 2026-02-20T21:06:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 -eq 0 2026-02-20T21:06:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:06:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:06:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ setup_pools cluster2 cluster1 2026-02-20T21:06:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:06:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_cluster=cluster1 2026-02-20T21:06:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mon_map_file 2026-02-20T21:06:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mon_addr 2026-02-20T21:06:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local admin_key_file 2026-02-20T21:06:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local uuid 2026-02-20T21:06:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:06:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-20T21:06:07.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror' already exists 2026-02-20T21:06:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:06:07.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-20T21:06:08.637 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:06:08.634+0000 7f5072251a40 -1 rbd::mirror::PoolReplayer: 0x55be02c5a900 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-20T21:06:08.637 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:06:08.634+0000 7fcf28f5ca40 -1 rbd::mirror::PoolReplayer: 0x55c45841ac00 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-20T21:06:08.638 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:06:08.635+0000 7fde3da71a40 -1 rbd::mirror::PoolReplayer: 0x56041294c900 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-20T21:06:08.638 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:06:08.635+0000 7f456d8b1a40 -1 rbd::mirror::PoolReplayer: 0x5617bf6bcc00 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-20T21:06:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror_parent' already exists 2026-02-20T21:06:08.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:06:08.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-20T21:06:09.635 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:06:09.633+0000 7f456d8b1a40 -1 rbd::mirror::PoolReplayer: 0x5617c0461800 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-20T21:06:09.635 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:06:09.633+0000 7fcf28f5ca40 -1 rbd::mirror::PoolReplayer: 0x55c4591ab800 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-20T21:06:09.635 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:06:09.633+0000 7fde3da71a40 -1 rbd::mirror::PoolReplayer: 0x5604136f1500 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-20T21:06:09.637 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:06:09.635+0000 7f5072251a40 -1 rbd::mirror::PoolReplayer: 0x55be03a0b500 init: failed to initialize remote pool poller: (2) No such file or directory 2026-02-20T21:06:11.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:06:11.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-20T21:06:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:06:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:06:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror pool 2026-02-20T21:06:14.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-20T21:06:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-20T21:06:14.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-20T21:06:14.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-20T21:06:14.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 pool 2026-02-20T21:06:14.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-20T21:06:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 pool 2026-02-20T21:06:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z ']' 2026-02-20T21:06:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z ']' 2026-02-20T21:06:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_add cluster2 mirror cluster1 2026-02-20T21:06:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:06:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local client_cluster=cluster1 2026-02-20T21:06:14.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_cluster=cluster1 2026-02-20T21:06:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local uuid_var_name 2026-02-20T21:06:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:06:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local error_code 2026-02-20T21:06:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local peer_uuid 2026-02-20T21:06:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 16 32 2026-02-20T21:06:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:06:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_uuid=b19e0387-9a8c-4ce0-9037-ecde016f05b9 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 17 ']' 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 -eq 0 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_add cluster2 mirror_parent cluster1 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror_parent 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local client_cluster=cluster1 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_cluster=cluster1 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local uuid_var_name 2026-02-20T21:06:14.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:06:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local error_code 2026-02-20T21:06:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local peer_uuid 2026-02-20T21:06:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 16 32 2026-02-20T21:06:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:06:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2026-02-20T21:06:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ peer_uuid=5e679c20-44a3-436b-85ec-ee9ecf7ffe81 2026-02-20T21:06:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-20T21:06:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:06:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 17 ']' 2026-02-20T21:06:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 -eq 0 2026-02-20T21:06:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:06:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:06:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:06:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirrors cluster1 2026-02-20T21:06:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:06:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:06:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster1:0 2026-02-20T21:06:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:06:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:06:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:06:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:06:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:06:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:06:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:06:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:06:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:06:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:06:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:06:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster1:1 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:06:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:06:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:06:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:06:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:06:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster1:2 2026-02-20T21:06:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:06:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:06:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:06:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:06:14.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster1:3 2026-02-20T21:06:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:06:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:06:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:06:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:06:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:06:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:06:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:06:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:06:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:06:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:06:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:06:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:06:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:06:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirrors cluster2 2026-02-20T21:06:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:06:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:06:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster2:0 2026-02-20T21:06:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:0 2026-02-20T21:06:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:06:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-20T21:06:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster2:0 2026-02-20T21:06:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:06:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:06:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster2 2026-02-20T21:06:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:06:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-20T21:06:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster2 2026-02-20T21:06:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster2 2026-02-20T21:06:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:06:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:06:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:06:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:06:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster2:1 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:1 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster2:1 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster2 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster2 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster2 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:06:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:06:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:06:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:06:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster2:2 2026-02-20T21:06:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:2 2026-02-20T21:06:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:06:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-20T21:06:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster2:2 2026-02-20T21:06:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:06:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:06:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster2 2026-02-20T21:06:14.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster2 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster2 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster2:3 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:3 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-20T21:06:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster2:3 2026-02-20T21:06:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:06:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:06:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster2 2026-02-20T21:06:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:06:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-20T21:06:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster2 2026-02-20T21:06:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster2 2026-02-20T21:06:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:06:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:06:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:06:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:06:14.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: add image and test replay after client crashes' 2026-02-20T21:06:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.IQNOxgiLiE/rbd-mirror.test.log 2026-02-20T21:06:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-20T21:06:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-20 21:06:14 TEST: add image and test replay after client crashes 2026-02-20T21:06:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-20 21:06:14 TEST: add image and test replay after client crashes 2026-02-20T21:06:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=test 2026-02-20T21:06:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror test journal 512M 2026-02-20T21:06:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:06:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:06:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror test 512M 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-20T21:06:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-20T21:06:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:06:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/test 2026-02-20T21:06:14.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:06:14.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:14.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:06:14.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:06:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:06:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:06:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:06:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:06:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:06:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:06:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:06:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:06:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:06:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:06:15.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:06:15.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:06:15.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:15.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:06:15.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:06:15.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:06:15.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:06:15.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:15.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:06:15.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:06:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:06:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:06:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:06:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:06:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:06:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:06:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:15.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:15.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:06:15.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:06:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:06:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:06:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:06:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:15.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:06:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:06:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:06:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:06:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:06:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:06:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:06:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:06:15.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:15.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:15.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:15.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:06:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:06:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:06:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:06:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:06:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:16.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:16.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:06:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:06:16.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:06:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:06:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:16.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:16.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:16.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:06:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:06:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:06:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:06:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:06:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:06:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:06:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:06:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:06:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:06:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:06:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:06:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:06:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:06:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:06:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:06:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 1 2026-02-20T21:06:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:06:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:06:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:17.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:17.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:17.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:17.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:17.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:17.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:17.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:17.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:17.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:06:17.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:17.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:06:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:06:17.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:06:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:17.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:17.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:06:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:06:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:06:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:06:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:06:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:06:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:06:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:06:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:06:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:06:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:06:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:06:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:06:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:17.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:06:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:17.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:06:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:06:17.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:06:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:06:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:06:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:06:17.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:06:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:06:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:06:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:06:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:06:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:06:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:06:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:06:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:06:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:06:17.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:17.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 2 2026-02-20T21:06:19.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:19.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:06:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:06:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:06:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:19.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:19.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:19.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:19.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:06:19.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:06:19.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:19.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:19.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:06:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:06:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:06:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:06:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:06:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:06:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:06:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:06:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:06:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:06:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:06:19.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:06:19.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:06:19.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:06:19.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:19.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:19.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:19.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:06:19.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:19.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:06:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:06:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:06:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:06:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:06:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:06:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:06:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:06:19.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:06:19.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:19.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:19.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:19.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:06:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:19.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:19.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 4 2026-02-20T21:06:23.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:23.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:06:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:06:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:06:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:23.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:06:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:06:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:23.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:06:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:06:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:23.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:23.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:23.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:23.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:23.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:23.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:23.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:23.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:23.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:06:23.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:06:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:06:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:06:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:06:23.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:06:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:06:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:06:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:06:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:06:23.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:23.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:23.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:23.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:23.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:23.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:23.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:23.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:06:23.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:06:23.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:06:23.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:23.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:23.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:23.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:06:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:06:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:06:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:06:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:06:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:06:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:06:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:06:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:06:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:06:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:06:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:06:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:06:23.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:23.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:23.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:23.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:06:23.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:23.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:23.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:23.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:06:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:06:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:06:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:06:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:06:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:23.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:06:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:06:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:06:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:06:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:06:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:06:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:06:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:06:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:06:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:06:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:23.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:06:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:23.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-20T21:06:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:06:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:06:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:06:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:06:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:31.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:31.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:31.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:31.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:06:31.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:06:31.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:06:31.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:31.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:31.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:31.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:31.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:31.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:31.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:06:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:06:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:06:31.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:31.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:31.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:31.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:06:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:31.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:06:31.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:06:31.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:06:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:31.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:06:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-20T21:06:38.639 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:06:38.637+0000 7f456d8b1a40 -1 rbd::mirror::Mirror: 0x5617be7dc240 update_pool_replayers: restarting failed pool replayer for uuid: 9d54e664-d69d-4c02-9066-950093e2d260 cluster: cluster2 client: client.mirror 2026-02-20T21:06:38.639 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:06:38.637+0000 7fde3da71a40 -1 rbd::mirror::Mirror: 0x560411a60240 update_pool_replayers: restarting failed pool replayer for uuid: 9d54e664-d69d-4c02-9066-950093e2d260 cluster: cluster2 client: client.mirror 2026-02-20T21:06:38.639 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:06:38.638+0000 7fcf28f5ca40 -1 rbd::mirror::Mirror: 0x55c457532240 update_pool_replayers: restarting failed pool replayer for uuid: 9d54e664-d69d-4c02-9066-950093e2d260 cluster: cluster2 client: client.mirror 2026-02-20T21:06:38.640 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:06:38.639+0000 7f5072251a40 -1 rbd::mirror::Mirror: 0x55be01d82240 update_pool_replayers: restarting failed pool replayer for uuid: 9d54e664-d69d-4c02-9066-950093e2d260 cluster: cluster2 client: client.mirror 2026-02-20T21:06:38.660 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:06:38.658+0000 7fde3da71a40 -1 rbd::mirror::Mirror: 0x560411a60240 update_pool_replayers: restarting failed pool replayer for uuid: 8e917922-9fe4-45e5-83d3-25635852754c cluster: cluster2 client: client.mirror 2026-02-20T21:06:38.662 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:06:38.660+0000 7f456d8b1a40 -1 rbd::mirror::Mirror: 0x5617be7dc240 update_pool_replayers: restarting failed pool replayer for uuid: 8e917922-9fe4-45e5-83d3-25635852754c cluster: cluster2 client: client.mirror 2026-02-20T21:06:38.663 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:06:38.661+0000 7f5072251a40 -1 rbd::mirror::Mirror: 0x55be01d82240 update_pool_replayers: restarting failed pool replayer for uuid: 8e917922-9fe4-45e5-83d3-25635852754c cluster: cluster2 client: client.mirror 2026-02-20T21:06:38.665 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:06:38.663+0000 7fcf28f5ca40 -1 rbd::mirror::Mirror: 0x55c457532240 update_pool_replayers: restarting failed pool replayer for uuid: 8e917922-9fe4-45e5-83d3-25635852754c cluster: cluster2 client: client.mirror 2026-02-20T21:06:40.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:40.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:06:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:06:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:06:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:40.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:06:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:06:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:06:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:06:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:40.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:40.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:40.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:06:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:06:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:06:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:06:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:06:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:06:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:06:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:06:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:40.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:06:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:06:40.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:06:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:40.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:06:40.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:40.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:40.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:40.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:06:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:06:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:06:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:06:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:06:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:06:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:06:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:06:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:06:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:06:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:06:40.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:06:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:40.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:40.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:40.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-20T21:06:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:06:48.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:06:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:06:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:06:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:06:48.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:06:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:06:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:06:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:06:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:06:48.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:06:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:06:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:06:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:06:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:06:48.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:06:48.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:06:48.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:48.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:48.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:06:48.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:06:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:06:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:06:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:06:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:06:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:06:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:06:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:06:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:06:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:06:48.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:06:48.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:48.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:48.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:48.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:06:48.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:06:48.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:48.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:48.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:06:48.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:06:48.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:06:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:06:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:06:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:06:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:06:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:06:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:06:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:06:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:06:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:06:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:06:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:06:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:06:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:06:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:06:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:06:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:06:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:06:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:06:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:06:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:06:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:06:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:06:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:06:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:06:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:06:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:06:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:06:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:06:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:06:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:06:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:06:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:06:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:06:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:06:48.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:48.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:48.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:06:48.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:06:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:06:48.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:06:48.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:06:48.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:06:48.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:06:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:06:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:06:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:06:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name= 2026-02-20T21:06:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 10 2026-02-20T21:06:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:06:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:06:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:06:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:06:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:06:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-20T21:06:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:06:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:06:48.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 1779712 bytes 2026-02-20T21:06:48.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 43675648 bytes 2026-02-20T21:06:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 109670400 bytes 2026-02-20T21:06:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 173492224 bytes 2026-02-20T21:06:50.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 214641664 bytes 2026-02-20T21:06:50.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 228742144 bytes 2026-02-20T21:06:50.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 251295744 bytes 2026-02-20T21:06:51.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 276073472 bytes 2026-02-20T21:06:51.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 325104640 bytes 2026-02-20T21:06:52.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 397299712 bytes 2026-02-20T21:06:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 430146560 bytes 2026-02-20T21:06:53.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 445383680 bytes 2026-02-20T21:06:54.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 517622784 bytes 2026-02-20T21:06:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 565163008 bytes 2026-02-20T21:06:54.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 594436096 bytes 2026-02-20T21:06:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 621070336 bytes 2026-02-20T21:06:55.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 657937408 bytes 2026-02-20T21:06:55.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 687443968 bytes 2026-02-20T21:06:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 690779136 bytes 2026-02-20T21:06:56.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 724288512 bytes 2026-02-20T21:06:56.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:06:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:06:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:06:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:06:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:06:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:06:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:06:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:06:56.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:06:56.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:06:56.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:06:56.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:06:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:06:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:06:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:06:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:06:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:06:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:06:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:06:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:06:57.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:06:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:06:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:06:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-20T21:06:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-20T21:06:57.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.2 on trial163 2026-02-20T21:06:57.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:06:41 2026-02-20T21:06:57.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:06:57.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:06:57.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: down+unknown 2026-02-20T21:06:57.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: status not found 2026-02-20T21:06:57.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: ' 2026-02-20T21:06:57.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:06:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-20T21:06:57.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-20T21:06:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-20T21:06:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:06:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:06:57.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-20T21:06:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:06:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:06:57.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.2 on trial163 2026-02-20T21:06:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-20T21:06:57.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:06:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:06:41' 2026-02-20T21:06:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:06:41' 2026-02-20T21:06:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:06:57.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:06:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:06:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-20 21:06:41' 2026-02-20T21:06:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:06:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:06:57.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:06:57.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='WARNINGOKWARNING142044287mirror.0trial16320.2.0-677-gf7870454falseOK42084286mirror.1trial16320.2.0-677-gf7870454falseOK42124282mirror.2trial16320.2.0-677-gf7870454trueOK42164285mirror.3trial16320.2.0-677-gf7870454falseOKtestba923525-490d-4a76-8e5f-b742509a94daup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42124282mirror.2trial1632026-02-20 21:06:41cluster28b968eaf-6c24-469f-bea5-c8acb780219edown+unknownstatus not found' 2026-02-20T21:06:57.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-20T21:06:57.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:06:57.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:06:41' 2026-02-20T21:06:57.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:06:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-20T21:06:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:06:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-20T21:06:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:06:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:06:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:06:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap1 2026-02-20T21:06:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap1 2026-02-20T21:06:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:06:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:06:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:06:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1 2026-02-20T21:06:57.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:07:06.054 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.3211219371966494) > weight (0.25) 2026-02-20T21:07:06.054 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.39898860958460336) > weight (0.25) 2026-02-20T21:07:06.054 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.48005498272716596) > weight (0.25) 2026-02-20T21:07:06.054 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.4322922530033725) > weight (0.25) 2026-02-20T21:07:06.054 INFO:tasks.rbd_mirror_thrash:waiting for 89.5753677253337 secs before thrashing 2026-02-20T21:07:12.510 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.4656782458689258) > weight (0.25) 2026-02-20T21:07:12.510 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9940817070168648) > weight (0.25) 2026-02-20T21:07:12.510 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-20T21:07:12.511 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-20T21:07:12.511 INFO:tasks.rbd_mirror_thrash:waiting for 2.1239036716082005 secs before reviving daemons 2026-02-20T21:07:12.512 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:07:12.510+0000 7fde3aa11640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 47141) UID: 1000 2026-02-20T21:07:14.635 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-20T21:07:14.636 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:07:20.737 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-20T21:07:20.737 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-20T21:07:20.737 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-20T21:07:20.740 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-20T21:07:20.740 INFO:tasks.rbd_mirror_thrash:waiting for 72.57389983129545 secs before thrashing 2026-02-20T21:07:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:07:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:07:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:07:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:07:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:07:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:07:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:07:29.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:07:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:07:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:07:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:07:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:07:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:07:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:07:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:07:29.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:07:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:07:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:07:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:07:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:07:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:07:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:07:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:07:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:07:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:07:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:07:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:29.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:07:29.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:29.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:29.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:07:29.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:07:29.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:07:29.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:07:29.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:29.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:29.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:29.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:29.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:07:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:07:29.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:07:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:07:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:07:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:29.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:07:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:07:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:07:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:07:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:07:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:07:29.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:07:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:07:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:30.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:30.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:07:30.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:07:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:07:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:07:30.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:07:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:07:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:07:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:07:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:07:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:07:30.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:07:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:07:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:07:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:07:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:07:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:07:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:30.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:30.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:07:30.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:07:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:07:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:07:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:07:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:07:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:07:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:07:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:07:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:07:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:30.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:30.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:07:30.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:07:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:07:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:07:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:07:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:07:30.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:07:30.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:07:30.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:30.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:07:30.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:07:30.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:07:30.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:07:30.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:07:30.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:30.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:07:30.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:07:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:07:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:07:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:07:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:07:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:07:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:07:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:30.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:30.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:30.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:30.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:30.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:30.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:30.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:30.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:30.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:30.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:30.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:30.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:30.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:30.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:30.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:30.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:30.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:30.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:30.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:30.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:07:30.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=31733], [object_number=28, tag_tid=2, entry_tid=31732], [object_number=31, tag_tid=2, entry_tid=31731], [object_number=30, tag_tid=2, entry_tid=31730]]], state=connected] 2026-02-20T21:07:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:30.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:30.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:30.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:30.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:30.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:30.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:30.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:30.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:30.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:30.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:30.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:30.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:30.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:30.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:30.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:30.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:07:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:30.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=31733], [object_number=28, tag_tid=2, entry_tid=31732], [object_number=31, tag_tid=2, entry_tid=31731], [object_number=30, tag_tid=2, entry_tid=31730]]], state=connected] 2026-02-20T21:07:30.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:30.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=31733' 2026-02-20T21:07:30.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=31733' 2026-02-20T21:07:30.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=2, entry_tid=31733' '!=' '' 2026-02-20T21:07:30.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=2, entry_tid=31733' = '' 2026-02-20T21:07:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=29, tag_tid=2, entry_tid=31733' 2026-02-20T21:07:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:30.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:30.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=31733' 2026-02-20T21:07:30.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:30.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:30.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:30.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:30.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=31733' 2026-02-20T21:07:30.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:30.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31733 2026-02-20T21:07:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 31733 2026-02-20T21:07:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:30.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:30.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:30.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:30.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:30.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:30.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:30.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:30.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:30.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:30.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:30.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:30.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:30.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:30.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:30.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:30.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:30.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:30.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:30.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:30.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:30.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:30.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:31.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:31.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:31.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:31.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:31.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:31.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:31.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:31.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:31.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:31.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:07:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=2, entry_tid=32350], [object_number=29, tag_tid=2, entry_tid=32349], [object_number=28, tag_tid=2, entry_tid=32348], [object_number=31, tag_tid=2, entry_tid=32347]]], state=connected] 2026-02-20T21:07:31.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:31.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:31.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:31.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:31.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:07:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=2, entry_tid=32350], [object_number=29, tag_tid=2, entry_tid=32349], [object_number=28, tag_tid=2, entry_tid=32348], [object_number=31, tag_tid=2, entry_tid=32347]]], state=connected] 2026-02-20T21:07:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=30, tag_tid=2, entry_tid=32350' 2026-02-20T21:07:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=30, tag_tid=2, entry_tid=32350' 2026-02-20T21:07:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=2, entry_tid=32350' '!=' 'object_number=29, tag_tid=2, entry_tid=31733' 2026-02-20T21:07:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=2, entry_tid=32350' = 'object_number=29, tag_tid=2, entry_tid=31733' 2026-02-20T21:07:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=30, tag_tid=2, entry_tid=32350' 2026-02-20T21:07:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=30, tag_tid=2, entry_tid=32350' 2026-02-20T21:07:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:31.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=30, tag_tid=2, entry_tid=32350' 2026-02-20T21:07:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:31.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32350 2026-02-20T21:07:31.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 32350 2026-02-20T21:07:31.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:31.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:31.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:31.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:31.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:31.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:31.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:31.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:31.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:31.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:31.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:31.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:31.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:31.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:31.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:31.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:31.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:31.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:31.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:31.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:31.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:31.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:31.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:31.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:31.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:31.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:31.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:31.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:31.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:31.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:31.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:31.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:31.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:31.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:31.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:31.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:31.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:31.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:31.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:31.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:31.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:31.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:31.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:31.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:31.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:31.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:31.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:31.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:31.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:31.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:31.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:07:31.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:31.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:31.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=28, tag_tid=2, entry_tid=32952], [object_number=31, tag_tid=2, entry_tid=32951], [object_number=30, tag_tid=2, entry_tid=32950], [object_number=29, tag_tid=2, entry_tid=32949]]], state=connected] 2026-02-20T21:07:31.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:31.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:31.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:31.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:31.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:31.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:31.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:31.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:31.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:31.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:31.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:07:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:31.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=28, tag_tid=2, entry_tid=32952], [object_number=31, tag_tid=2, entry_tid=32951], [object_number=30, tag_tid=2, entry_tid=32950], [object_number=29, tag_tid=2, entry_tid=32949]]], state=connected] 2026-02-20T21:07:31.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:31.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=28, tag_tid=2, entry_tid=32952' 2026-02-20T21:07:31.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=28, tag_tid=2, entry_tid=32952' 2026-02-20T21:07:31.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=28, tag_tid=2, entry_tid=32952' '!=' 'object_number=30, tag_tid=2, entry_tid=32350' 2026-02-20T21:07:31.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=28, tag_tid=2, entry_tid=32952' = 'object_number=30, tag_tid=2, entry_tid=32350' 2026-02-20T21:07:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=28, tag_tid=2, entry_tid=32952' 2026-02-20T21:07:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:31.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:31.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=28, tag_tid=2, entry_tid=32952' 2026-02-20T21:07:31.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:31.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:31.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:31.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:31.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:31.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:31.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=28, tag_tid=2, entry_tid=32952' 2026-02-20T21:07:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32952 2026-02-20T21:07:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 32952 2026-02-20T21:07:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:31.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:31.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:31.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:31.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:31.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:31.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:31.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:31.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:31.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:31.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:31.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:31.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:31.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:31.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:31.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:31.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:31.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:31.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:31.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:31.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:31.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:31.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:31.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:31.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:31.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:32.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:32.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:32.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:32.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:32.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:07:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=33499], [object_number=30, tag_tid=2, entry_tid=33498], [object_number=29, tag_tid=2, entry_tid=33497], [object_number=28, tag_tid=2, entry_tid=33496]]], state=connected] 2026-02-20T21:07:32.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:32.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:32.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:32.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:32.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:32.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:07:32.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=33499], [object_number=30, tag_tid=2, entry_tid=33498], [object_number=29, tag_tid=2, entry_tid=33497], [object_number=28, tag_tid=2, entry_tid=33496]]], state=connected] 2026-02-20T21:07:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=35, tag_tid=2, entry_tid=33499' 2026-02-20T21:07:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=35, tag_tid=2, entry_tid=33499' 2026-02-20T21:07:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=33499' '!=' 'object_number=28, tag_tid=2, entry_tid=32952' 2026-02-20T21:07:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=33499' = 'object_number=28, tag_tid=2, entry_tid=32952' 2026-02-20T21:07:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=35, tag_tid=2, entry_tid=33499' 2026-02-20T21:07:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:32.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=33499' 2026-02-20T21:07:32.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:32.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:32.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:32.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:32.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=33499' 2026-02-20T21:07:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:32.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33499 2026-02-20T21:07:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 33499 2026-02-20T21:07:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:32.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:32.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:32.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:32.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:32.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:32.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:32.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:32.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:32.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:32.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:32.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:32.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:32.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:32.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:32.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:32.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:32.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:32.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:32.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:32.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:32.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:32.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:32.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:32.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:32.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:32.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:32.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:32.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:32.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:32.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:32.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:32.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:32.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:32.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:32.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:32.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:32.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:32.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:32.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:32.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=34053], [object_number=32, tag_tid=2, entry_tid=34052], [object_number=35, tag_tid=2, entry_tid=34051], [object_number=34, tag_tid=2, entry_tid=34050]]], state=connected] 2026-02-20T21:07:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:32.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:32.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=34053], [object_number=32, tag_tid=2, entry_tid=34052], [object_number=35, tag_tid=2, entry_tid=34051], [object_number=34, tag_tid=2, entry_tid=34050]]], state=connected] 2026-02-20T21:07:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=34053' 2026-02-20T21:07:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=34053' 2026-02-20T21:07:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=34053' '!=' 'object_number=35, tag_tid=2, entry_tid=33499' 2026-02-20T21:07:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=34053' = 'object_number=35, tag_tid=2, entry_tid=33499' 2026-02-20T21:07:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=2, entry_tid=34053' 2026-02-20T21:07:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=34053' 2026-02-20T21:07:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=34053' 2026-02-20T21:07:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34053 2026-02-20T21:07:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 34053 2026-02-20T21:07:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:32.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:32.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:32.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:32.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:32.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:32.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:32.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:32.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:32.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:32.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:32.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:32.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:32.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:32.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:32.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:32.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:32.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:32.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:32.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:32.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:32.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:32.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:32.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:32.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:32.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:32.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:32.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:32.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:32.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:32.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:32.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:32.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:32.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:32.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:32.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:32.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:32.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:32.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:32.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:32.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:32.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:32.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:32.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:32.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:32.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:32.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:32.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:32.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:32.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:32.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:32.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:32.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:32.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:32.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:32.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:32.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:32.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:32.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:32.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:33.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:33.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:33.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:33.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:33.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:33.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:33.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:33.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:33.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:33.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:33.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:33.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:33.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:33.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:33.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:33.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:33.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:33.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:33.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:33.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=34609], [object_number=32, tag_tid=2, entry_tid=34608], [object_number=35, tag_tid=2, entry_tid=34607], [object_number=34, tag_tid=2, entry_tid=34606]]], state=connected] 2026-02-20T21:07:33.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:33.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:33.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:33.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:33.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:33.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:33.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=34609], [object_number=32, tag_tid=2, entry_tid=34608], [object_number=35, tag_tid=2, entry_tid=34607], [object_number=34, tag_tid=2, entry_tid=34606]]], state=connected] 2026-02-20T21:07:33.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=34609' 2026-02-20T21:07:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=34609' 2026-02-20T21:07:33.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=34609' '!=' 'object_number=33, tag_tid=2, entry_tid=34053' 2026-02-20T21:07:33.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:33.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=34609' = 'object_number=33, tag_tid=2, entry_tid=34053' 2026-02-20T21:07:33.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=2, entry_tid=34609' 2026-02-20T21:07:33.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:33.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:33.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=34609' 2026-02-20T21:07:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:33.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:33.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:33.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:33.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=34609' 2026-02-20T21:07:33.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:33.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34609 2026-02-20T21:07:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 34609 2026-02-20T21:07:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:33.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:33.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:33.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:33.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:33.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:33.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:33.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:33.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:33.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:33.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:33.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:33.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:33.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:33.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:33.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:33.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:33.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:33.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:33.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:33.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:33.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:33.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:33.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:33.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:33.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:33.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:33.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:33.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:33.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:33.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:33.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:33.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:33.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:33.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:33.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:33.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:33.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:33.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:33.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:33.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:33.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:33.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:33.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:33.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:33.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:33.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:33.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:33.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:33.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:33.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:33.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:33.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:33.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:33.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:33.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:33.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:33.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:33.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:33.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=35237], [object_number=32, tag_tid=2, entry_tid=35236], [object_number=35, tag_tid=2, entry_tid=35235], [object_number=34, tag_tid=2, entry_tid=35234]]], state=connected] 2026-02-20T21:07:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:33.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:33.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=35237], [object_number=32, tag_tid=2, entry_tid=35236], [object_number=35, tag_tid=2, entry_tid=35235], [object_number=34, tag_tid=2, entry_tid=35234]]], state=connected] 2026-02-20T21:07:33.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=35237' 2026-02-20T21:07:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=35237' 2026-02-20T21:07:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=35237' '!=' 'object_number=33, tag_tid=2, entry_tid=34609' 2026-02-20T21:07:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=35237' = 'object_number=33, tag_tid=2, entry_tid=34609' 2026-02-20T21:07:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=2, entry_tid=35237' 2026-02-20T21:07:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:33.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=35237' 2026-02-20T21:07:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:33.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:33.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:33.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:33.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:33.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:33.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=35237' 2026-02-20T21:07:33.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:33.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:33.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35237 2026-02-20T21:07:33.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 35237 2026-02-20T21:07:33.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:33.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:33.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:33.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:33.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:33.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:33.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:33.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:33.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:33.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:33.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:33.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:33.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:33.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:33.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:33.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:33.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:33.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:33.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:33.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:33.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:33.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:33.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:33.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:33.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:33.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:33.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:33.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:33.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:33.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:33.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:33.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:33.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:33.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:33.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:33.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:33.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:33.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:33.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:33.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:33.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:33.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:33.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:33.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:33.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=35571], [object_number=34, tag_tid=2, entry_tid=35570], [object_number=33, tag_tid=2, entry_tid=35569], [object_number=32, tag_tid=2, entry_tid=35568]]], state=connected] 2026-02-20T21:07:34.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:34.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:34.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:34.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:34.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:34.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:34.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:34.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:34.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:34.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:34.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:34.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:34.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:34.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:34.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:34.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:34.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=35571], [object_number=34, tag_tid=2, entry_tid=35570], [object_number=33, tag_tid=2, entry_tid=35569], [object_number=32, tag_tid=2, entry_tid=35568]]], state=connected] 2026-02-20T21:07:34.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=35, tag_tid=2, entry_tid=35571' 2026-02-20T21:07:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=35, tag_tid=2, entry_tid=35571' 2026-02-20T21:07:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=35571' '!=' 'object_number=33, tag_tid=2, entry_tid=35237' 2026-02-20T21:07:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=35571' = 'object_number=33, tag_tid=2, entry_tid=35237' 2026-02-20T21:07:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=35, tag_tid=2, entry_tid=35571' 2026-02-20T21:07:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:34.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:34.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=35571' 2026-02-20T21:07:34.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:34.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:34.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:34.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:34.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:34.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:34.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=35571' 2026-02-20T21:07:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:34.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35571 2026-02-20T21:07:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 35571 2026-02-20T21:07:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:34.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:34.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:34.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:34.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:34.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:34.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:34.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:34.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:34.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:34.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:34.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:34.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:34.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:34.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:34.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:34.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:34.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:34.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:34.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:34.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:34.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:34.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:34.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:34.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:34.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:34.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:34.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:34.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:34.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:34.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:34.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:34.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:34.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:34.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:34.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:34.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:34.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:34.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:34.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:34.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:34.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:34.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:34.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:34.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:34.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:34.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:34.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:34.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:34.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:34.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:34.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:34.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:34.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:34.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:34.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:34.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:34.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:34.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:34.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:34.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:34.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:34.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:34.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:34.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:34.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:34.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:34.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:34.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:34.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:34.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:34.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:34.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:34.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:34.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:34.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:34.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:34.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=36129], [object_number=32, tag_tid=2, entry_tid=36128], [object_number=35, tag_tid=2, entry_tid=36127], [object_number=34, tag_tid=2, entry_tid=36126]]], state=connected] 2026-02-20T21:07:34.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:34.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:34.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=36129], [object_number=32, tag_tid=2, entry_tid=36128], [object_number=35, tag_tid=2, entry_tid=36127], [object_number=34, tag_tid=2, entry_tid=36126]]], state=connected] 2026-02-20T21:07:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=36129' 2026-02-20T21:07:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=36129' 2026-02-20T21:07:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=36129' '!=' 'object_number=35, tag_tid=2, entry_tid=35571' 2026-02-20T21:07:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=36129' = 'object_number=35, tag_tid=2, entry_tid=35571' 2026-02-20T21:07:34.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=2, entry_tid=36129' 2026-02-20T21:07:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:34.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:34.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=36129' 2026-02-20T21:07:34.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:34.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:34.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:34.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:34.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:34.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:34.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:34.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=36129' 2026-02-20T21:07:34.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:34.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36129 2026-02-20T21:07:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 36129 2026-02-20T21:07:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:34.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:34.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:34.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:34.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:34.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:34.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:34.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:34.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:34.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:34.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:34.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:34.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:34.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:34.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:34.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:34.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:34.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:34.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:34.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:34.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:34.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:34.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:34.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:34.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:34.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:34.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:34.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:34.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:34.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:34.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:34.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:34.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:34.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:34.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:34.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:34.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:34.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:34.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:34.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:34.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:34.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:34.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:34.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:34.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:34.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:35.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:35.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:35.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:35.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:35.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:35.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:35.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:35.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:35.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=36769], [object_number=32, tag_tid=2, entry_tid=36768], [object_number=35, tag_tid=2, entry_tid=36767], [object_number=34, tag_tid=2, entry_tid=36766]]], state=connected] 2026-02-20T21:07:35.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:35.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:35.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:35.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:35.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=36769], [object_number=32, tag_tid=2, entry_tid=36768], [object_number=35, tag_tid=2, entry_tid=36767], [object_number=34, tag_tid=2, entry_tid=36766]]], state=connected] 2026-02-20T21:07:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=36769' 2026-02-20T21:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=36769' 2026-02-20T21:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=36769' '!=' 'object_number=33, tag_tid=2, entry_tid=36129' 2026-02-20T21:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=2, entry_tid=36769' = 'object_number=33, tag_tid=2, entry_tid=36129' 2026-02-20T21:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=2, entry_tid=36769' 2026-02-20T21:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:35.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:35.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=36769' 2026-02-20T21:07:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:35.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:35.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=36769' 2026-02-20T21:07:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36769 2026-02-20T21:07:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 36769 2026-02-20T21:07:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:35.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:35.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:35.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:35.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:35.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:35.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:35.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:35.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:35.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:35.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:35.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:35.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:35.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:35.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:35.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:35.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:35.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:35.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:35.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:35.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:35.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:35.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:35.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:35.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:35.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:35.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:35.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:35.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:35.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:35.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:35.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:35.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:35.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:35.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:35.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:35.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:35.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=37367], [object_number=34, tag_tid=2, entry_tid=37366], [object_number=33, tag_tid=2, entry_tid=37365], [object_number=32, tag_tid=2, entry_tid=37364]]], state=connected] 2026-02-20T21:07:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:07:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=2, entry_tid=37367], [object_number=34, tag_tid=2, entry_tid=37366], [object_number=33, tag_tid=2, entry_tid=37365], [object_number=32, tag_tid=2, entry_tid=37364]]], state=connected] 2026-02-20T21:07:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=35, tag_tid=2, entry_tid=37367' 2026-02-20T21:07:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=35, tag_tid=2, entry_tid=37367' 2026-02-20T21:07:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=37367' '!=' 'object_number=33, tag_tid=2, entry_tid=36769' 2026-02-20T21:07:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=35, tag_tid=2, entry_tid=37367' = 'object_number=33, tag_tid=2, entry_tid=36769' 2026-02-20T21:07:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=35, tag_tid=2, entry_tid=37367' 2026-02-20T21:07:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=37367' 2026-02-20T21:07:35.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:35.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:35.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=35, tag_tid=2, entry_tid=37367' 2026-02-20T21:07:35.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:35.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:35.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37367 2026-02-20T21:07:35.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 37367 2026-02-20T21:07:35.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:35.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:35.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:35.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:35.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:35.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:35.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:35.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:35.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:35.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:35.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:35.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:35.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:35.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:35.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:35.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:35.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:35.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:35.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:35.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:35.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:35.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:35.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:35.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:35.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:35.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:35.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:35.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:35.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:35.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:35.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:35.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:35.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:35.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:35.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:35.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:35.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:35.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:35.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:36.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:36.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:36.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:36.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:36.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:36.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:36.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:36.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:36.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:36.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:36.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:36.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:36.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:36.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:36.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:36.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:36.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=37925], [object_number=36, tag_tid=2, entry_tid=37924], [object_number=39, tag_tid=2, entry_tid=37923], [object_number=38, tag_tid=2, entry_tid=37922]]], state=connected] 2026-02-20T21:07:36.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:36.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:36.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:36.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:36.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=37925], [object_number=36, tag_tid=2, entry_tid=37924], [object_number=39, tag_tid=2, entry_tid=37923], [object_number=38, tag_tid=2, entry_tid=37922]]], state=connected] 2026-02-20T21:07:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:36.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=37925' 2026-02-20T21:07:36.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=37925' 2026-02-20T21:07:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=37925' '!=' 'object_number=35, tag_tid=2, entry_tid=37367' 2026-02-20T21:07:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=37925' = 'object_number=35, tag_tid=2, entry_tid=37367' 2026-02-20T21:07:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=37, tag_tid=2, entry_tid=37925' 2026-02-20T21:07:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=37925' 2026-02-20T21:07:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=37925' 2026-02-20T21:07:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37925 2026-02-20T21:07:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 37925 2026-02-20T21:07:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:36.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:36.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:36.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:36.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:36.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:36.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:36.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:36.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:36.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:36.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:36.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:36.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:36.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:36.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:36.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:36.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:36.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:36.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:36.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:36.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:36.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:36.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:36.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:36.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:36.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:36.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:36.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:36.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:36.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:36.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:36.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:36.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:36.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:36.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:36.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:36.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:36.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:36.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:36.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:36.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:36.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:36.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=38550], [object_number=37, tag_tid=2, entry_tid=38549], [object_number=36, tag_tid=2, entry_tid=38548], [object_number=39, tag_tid=2, entry_tid=38547]]], state=connected] 2026-02-20T21:07:36.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:36.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:36.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:36.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:36.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:36.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:36.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:36.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:36.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:36.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:36.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:36.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:36.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:36.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:36.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:36.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=38, tag_tid=2, entry_tid=38550], [object_number=37, tag_tid=2, entry_tid=38549], [object_number=36, tag_tid=2, entry_tid=38548], [object_number=39, tag_tid=2, entry_tid=38547]]], state=connected] 2026-02-20T21:07:36.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:36.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=38, tag_tid=2, entry_tid=38550' 2026-02-20T21:07:36.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=38, tag_tid=2, entry_tid=38550' 2026-02-20T21:07:36.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=38550' '!=' 'object_number=37, tag_tid=2, entry_tid=37925' 2026-02-20T21:07:36.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:36.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=2, entry_tid=38550' = 'object_number=37, tag_tid=2, entry_tid=37925' 2026-02-20T21:07:36.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=38, tag_tid=2, entry_tid=38550' 2026-02-20T21:07:36.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:36.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:36.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:36.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:36.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=38550' 2026-02-20T21:07:36.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:36.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:36.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:36.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:36.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:36.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:36.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:36.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=38, tag_tid=2, entry_tid=38550' 2026-02-20T21:07:36.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:36.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:36.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38550 2026-02-20T21:07:36.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 38550 2026-02-20T21:07:36.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:36.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:36.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:36.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:36.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:36.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:36.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:36.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:36.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:36.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:36.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:36.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:36.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:36.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:36.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:36.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:36.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:36.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:36.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:36.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:36.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:36.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:36.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:36.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:36.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:36.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:36.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:36.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:36.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:36.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:36.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:36.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:36.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:36.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:36.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:36.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:36.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:36.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:36.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:36.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:36.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:36.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:36.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:36.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:36.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:36.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:36.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:36.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:36.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:36.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:36.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:36.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:36.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:37.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:37.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:37.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:37.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:37.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:37.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:37.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:37.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:37.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:37.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=39169], [object_number=36, tag_tid=2, entry_tid=39168], [object_number=39, tag_tid=2, entry_tid=39167], [object_number=38, tag_tid=2, entry_tid=39166]]], state=connected] 2026-02-20T21:07:37.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:37.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:37.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:37.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:37.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:37.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:37.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:37.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:37.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:37.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=39169], [object_number=36, tag_tid=2, entry_tid=39168], [object_number=39, tag_tid=2, entry_tid=39167], [object_number=38, tag_tid=2, entry_tid=39166]]], state=connected] 2026-02-20T21:07:37.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=39169' 2026-02-20T21:07:37.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=39169' 2026-02-20T21:07:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=39169' '!=' 'object_number=38, tag_tid=2, entry_tid=38550' 2026-02-20T21:07:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=37, tag_tid=2, entry_tid=39169' = 'object_number=38, tag_tid=2, entry_tid=38550' 2026-02-20T21:07:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=37, tag_tid=2, entry_tid=39169' 2026-02-20T21:07:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:37.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:37.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=39169' 2026-02-20T21:07:37.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:37.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:37.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:37.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:37.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:37.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=39169' 2026-02-20T21:07:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:37.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39169 2026-02-20T21:07:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 39169 2026-02-20T21:07:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:37.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:37.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:37.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:37.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:37.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:37.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:37.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:37.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:37.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:37.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:37.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:37.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:37.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:37.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:37.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:37.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:37.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:37.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:37.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:37.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:37.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:37.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:37.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:37.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:37.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:37.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:37.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:37.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:37.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:37.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=39728], [object_number=39, tag_tid=2, entry_tid=39727], [object_number=38, tag_tid=2, entry_tid=39726], [object_number=37, tag_tid=2, entry_tid=39725]]], state=connected] 2026-02-20T21:07:37.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:37.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:37.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:37.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:37.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:37.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=39728], [object_number=39, tag_tid=2, entry_tid=39727], [object_number=38, tag_tid=2, entry_tid=39726], [object_number=37, tag_tid=2, entry_tid=39725]]], state=connected] 2026-02-20T21:07:37.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=36, tag_tid=2, entry_tid=39728' 2026-02-20T21:07:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=36, tag_tid=2, entry_tid=39728' 2026-02-20T21:07:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=39728' '!=' 'object_number=37, tag_tid=2, entry_tid=39169' 2026-02-20T21:07:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=39728' = 'object_number=37, tag_tid=2, entry_tid=39169' 2026-02-20T21:07:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=36, tag_tid=2, entry_tid=39728' 2026-02-20T21:07:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:37.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:37.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=39728' 2026-02-20T21:07:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:37.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:37.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:37.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:37.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=39728' 2026-02-20T21:07:37.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39728 2026-02-20T21:07:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 39728 2026-02-20T21:07:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:37.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:37.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:37.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:37.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:37.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:37.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:37.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:37.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:37.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:37.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:37.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:37.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:37.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:37.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:37.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:37.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:37.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:37.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:37.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:37.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:37.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:37.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:37.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:37.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:37.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:37.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:37.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:37.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:37.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:37.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:37.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:37.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:37.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:38.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:38.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:38.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:38.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:38.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:38.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:38.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:38.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:38.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:38.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:38.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:38.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:38.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:38.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=40299], [object_number=38, tag_tid=2, entry_tid=40298], [object_number=37, tag_tid=2, entry_tid=40297], [object_number=36, tag_tid=2, entry_tid=40296]]], state=connected] 2026-02-20T21:07:38.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:38.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:38.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:38.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:38.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:38.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:38.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:38.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=39, tag_tid=2, entry_tid=40299], [object_number=38, tag_tid=2, entry_tid=40298], [object_number=37, tag_tid=2, entry_tid=40297], [object_number=36, tag_tid=2, entry_tid=40296]]], state=connected] 2026-02-20T21:07:38.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:38.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=39, tag_tid=2, entry_tid=40299' 2026-02-20T21:07:38.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=39, tag_tid=2, entry_tid=40299' 2026-02-20T21:07:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=40299' '!=' 'object_number=36, tag_tid=2, entry_tid=39728' 2026-02-20T21:07:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=39, tag_tid=2, entry_tid=40299' = 'object_number=36, tag_tid=2, entry_tid=39728' 2026-02-20T21:07:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=39, tag_tid=2, entry_tid=40299' 2026-02-20T21:07:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=40299' 2026-02-20T21:07:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=39, tag_tid=2, entry_tid=40299' 2026-02-20T21:07:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40299 2026-02-20T21:07:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 40299 2026-02-20T21:07:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:38.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:38.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:38.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:38.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:38.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:38.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:38.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:38.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:38.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:38.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:38.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:38.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:38.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:38.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:38.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:38.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:38.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:38.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:38.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:38.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:38.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:38.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:38.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:38.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:38.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:38.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:38.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:38.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:38.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:38.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:38.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:38.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:38.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:38.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:38.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:38.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:38.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:38.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=40880], [object_number=39, tag_tid=2, entry_tid=40879], [object_number=38, tag_tid=2, entry_tid=40878], [object_number=37, tag_tid=2, entry_tid=40877]]], state=connected] 2026-02-20T21:07:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:38.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:38.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:38.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:38.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:38.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=40880], [object_number=39, tag_tid=2, entry_tid=40879], [object_number=38, tag_tid=2, entry_tid=40878], [object_number=37, tag_tid=2, entry_tid=40877]]], state=connected] 2026-02-20T21:07:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=36, tag_tid=2, entry_tid=40880' 2026-02-20T21:07:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=36, tag_tid=2, entry_tid=40880' 2026-02-20T21:07:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=40880' '!=' 'object_number=39, tag_tid=2, entry_tid=40299' 2026-02-20T21:07:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=40880' = 'object_number=39, tag_tid=2, entry_tid=40299' 2026-02-20T21:07:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=36, tag_tid=2, entry_tid=40880' 2026-02-20T21:07:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=40880' 2026-02-20T21:07:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:38.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=40880' 2026-02-20T21:07:38.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:38.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40880 2026-02-20T21:07:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 40880 2026-02-20T21:07:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:38.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:38.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:38.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:38.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:38.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:38.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:38.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:38.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:38.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:38.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:38.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:38.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:38.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:38.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:38.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:39.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=41196], [object_number=39, tag_tid=2, entry_tid=41195], [object_number=38, tag_tid=2, entry_tid=41194], [object_number=37, tag_tid=2, entry_tid=41193]]], state=connected] 2026-02-20T21:07:39.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:39.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:39.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:39.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:39.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:39.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:39.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:39.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:39.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:39.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:39.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:39.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:39.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:39.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:39.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:39.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=36, tag_tid=2, entry_tid=41196], [object_number=39, tag_tid=2, entry_tid=41195], [object_number=38, tag_tid=2, entry_tid=41194], [object_number=37, tag_tid=2, entry_tid=41193]]], state=connected] 2026-02-20T21:07:39.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=36, tag_tid=2, entry_tid=41196' 2026-02-20T21:07:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=36, tag_tid=2, entry_tid=41196' 2026-02-20T21:07:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=41196' '!=' 'object_number=36, tag_tid=2, entry_tid=40880' 2026-02-20T21:07:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=36, tag_tid=2, entry_tid=41196' = 'object_number=36, tag_tid=2, entry_tid=40880' 2026-02-20T21:07:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=36, tag_tid=2, entry_tid=41196' 2026-02-20T21:07:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=41196' 2026-02-20T21:07:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:39.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:39.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:39.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:39.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=36, tag_tid=2, entry_tid=41196' 2026-02-20T21:07:39.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:39.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41196 2026-02-20T21:07:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 41196 2026-02-20T21:07:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:39.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:39.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:39.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:39.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:39.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:39.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:39.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:39.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:39.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:39.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:39.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:39.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:39.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:39.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:39.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:39.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:39.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:39.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:39.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:39.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:39.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:39.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:39.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:39.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:39.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:39.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:39.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:39.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:39.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:39.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:39.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:39.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:39.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:39.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:39.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:39.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:39.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:39.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:39.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:39.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:39.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:39.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:39.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:39.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:39.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:39.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:39.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:39.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:39.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:39.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:39.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:39.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:39.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:39.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:39.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:39.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:39.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=41869], [object_number=40, tag_tid=2, entry_tid=41868], [object_number=39, tag_tid=2, entry_tid=41867], [object_number=38, tag_tid=2, entry_tid=41866]]], state=connected] 2026-02-20T21:07:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:39.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:40.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:40.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:40.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:07:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=41869], [object_number=40, tag_tid=2, entry_tid=41868], [object_number=39, tag_tid=2, entry_tid=41867], [object_number=38, tag_tid=2, entry_tid=41866]]], state=connected] 2026-02-20T21:07:40.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=41869' 2026-02-20T21:07:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=41869' 2026-02-20T21:07:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=2, entry_tid=41869' '!=' 'object_number=36, tag_tid=2, entry_tid=41196' 2026-02-20T21:07:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=2, entry_tid=41869' = 'object_number=36, tag_tid=2, entry_tid=41196' 2026-02-20T21:07:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=41, tag_tid=2, entry_tid=41869' 2026-02-20T21:07:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:40.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:40.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=41869' 2026-02-20T21:07:40.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:40.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:40.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:40.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:40.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:40.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:40.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=41869' 2026-02-20T21:07:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41869 2026-02-20T21:07:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 41869 2026-02-20T21:07:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:40.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:40.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:40.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:40.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:40.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:40.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:40.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:40.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:40.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:40.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:40.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:40.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:40.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:40.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:40.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:40.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:40.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:40.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:40.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:40.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:40.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:40.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:40.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:40.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:40.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:40.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:40.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:40.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:40.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:40.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:40.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:40.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:40.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:40.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:40.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:40.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:40.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:40.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:40.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:40.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:40.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:40.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:40.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:40.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:40.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:40.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:40.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:40.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:40.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:40.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:40.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:40.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:40.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:40.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:40.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:40.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:40.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:40.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:40.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:40.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=40, tag_tid=2, entry_tid=42392], [object_number=43, tag_tid=2, entry_tid=42391], [object_number=42, tag_tid=2, entry_tid=42390], [object_number=41, tag_tid=2, entry_tid=42389]]], state=connected] 2026-02-20T21:07:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:40.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:40.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:40.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:40.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=40, tag_tid=2, entry_tid=42392], [object_number=43, tag_tid=2, entry_tid=42391], [object_number=42, tag_tid=2, entry_tid=42390], [object_number=41, tag_tid=2, entry_tid=42389]]], state=connected] 2026-02-20T21:07:40.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=2, entry_tid=42392' 2026-02-20T21:07:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=40, tag_tid=2, entry_tid=42392' 2026-02-20T21:07:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=2, entry_tid=42392' '!=' 'object_number=41, tag_tid=2, entry_tid=41869' 2026-02-20T21:07:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=2, entry_tid=42392' = 'object_number=41, tag_tid=2, entry_tid=41869' 2026-02-20T21:07:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=40, tag_tid=2, entry_tid=42392' 2026-02-20T21:07:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:40.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:40.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=2, entry_tid=42392' 2026-02-20T21:07:40.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:40.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:40.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:40.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:40.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:40.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:40.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=2, entry_tid=42392' 2026-02-20T21:07:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42392 2026-02-20T21:07:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 42392 2026-02-20T21:07:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:40.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:40.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:40.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:40.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:40.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:40.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:40.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:40.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:40.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:40.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:40.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:40.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:40.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:40.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:40.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:40.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:40.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:40.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:40.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:40.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:40.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:40.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:40.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:40.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:40.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:40.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:40.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:40.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:40.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=42951], [object_number=42, tag_tid=2, entry_tid=42950], [object_number=41, tag_tid=2, entry_tid=42949], [object_number=40, tag_tid=2, entry_tid=42948]]], state=connected] 2026-02-20T21:07:40.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:40.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:41.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=42951], [object_number=42, tag_tid=2, entry_tid=42950], [object_number=41, tag_tid=2, entry_tid=42949], [object_number=40, tag_tid=2, entry_tid=42948]]], state=connected] 2026-02-20T21:07:41.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=2, entry_tid=42951' 2026-02-20T21:07:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=43, tag_tid=2, entry_tid=42951' 2026-02-20T21:07:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=42951' '!=' 'object_number=40, tag_tid=2, entry_tid=42392' 2026-02-20T21:07:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=42951' = 'object_number=40, tag_tid=2, entry_tid=42392' 2026-02-20T21:07:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=43, tag_tid=2, entry_tid=42951' 2026-02-20T21:07:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=42951' 2026-02-20T21:07:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:41.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:41.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:41.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=42951' 2026-02-20T21:07:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:41.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42951 2026-02-20T21:07:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 42951 2026-02-20T21:07:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:41.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:41.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:41.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:41.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:41.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:41.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:41.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:41.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:41.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:41.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:41.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:41.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:41.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:41.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:41.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:41.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:41.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=43551], [object_number=42, tag_tid=2, entry_tid=43550], [object_number=41, tag_tid=2, entry_tid=43549], [object_number=40, tag_tid=2, entry_tid=43548]]], state=connected] 2026-02-20T21:07:41.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:41.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=43551], [object_number=42, tag_tid=2, entry_tid=43550], [object_number=41, tag_tid=2, entry_tid=43549], [object_number=40, tag_tid=2, entry_tid=43548]]], state=connected] 2026-02-20T21:07:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=2, entry_tid=43551' 2026-02-20T21:07:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=43, tag_tid=2, entry_tid=43551' 2026-02-20T21:07:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=43551' '!=' 'object_number=43, tag_tid=2, entry_tid=42951' 2026-02-20T21:07:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=43551' = 'object_number=43, tag_tid=2, entry_tid=42951' 2026-02-20T21:07:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=43, tag_tid=2, entry_tid=43551' 2026-02-20T21:07:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=43551' 2026-02-20T21:07:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:41.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=43551' 2026-02-20T21:07:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43551 2026-02-20T21:07:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 43551 2026-02-20T21:07:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:41.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:41.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:41.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:41.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:41.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:41.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:41.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:41.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:41.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:41.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:41.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:41.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:41.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:41.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:41.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:41.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:41.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:41.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:41.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:41.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:41.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:41.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:41.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:41.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:41.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:41.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:41.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:41.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:41.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:41.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:41.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:41.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:41.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:41.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:41.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:41.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:41.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:41.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:41.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:41.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:41.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:41.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:41.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:41.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:41.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:41.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:41.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:41.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:41.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:41.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=44159], [object_number=42, tag_tid=2, entry_tid=44158], [object_number=41, tag_tid=2, entry_tid=44157], [object_number=40, tag_tid=2, entry_tid=44156]]], state=connected] 2026-02-20T21:07:41.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:41.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:41.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:41.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:41.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:41.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:41.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:41.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:41.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:41.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:41.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:41.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=43, tag_tid=2, entry_tid=44159], [object_number=42, tag_tid=2, entry_tid=44158], [object_number=41, tag_tid=2, entry_tid=44157], [object_number=40, tag_tid=2, entry_tid=44156]]], state=connected] 2026-02-20T21:07:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=43, tag_tid=2, entry_tid=44159' 2026-02-20T21:07:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=43, tag_tid=2, entry_tid=44159' 2026-02-20T21:07:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=44159' '!=' 'object_number=43, tag_tid=2, entry_tid=43551' 2026-02-20T21:07:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=43, tag_tid=2, entry_tid=44159' = 'object_number=43, tag_tid=2, entry_tid=43551' 2026-02-20T21:07:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=43, tag_tid=2, entry_tid=44159' 2026-02-20T21:07:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:42.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=44159' 2026-02-20T21:07:42.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:42.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:42.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:42.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=43, tag_tid=2, entry_tid=44159' 2026-02-20T21:07:42.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:42.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:42.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44159 2026-02-20T21:07:42.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 44159 2026-02-20T21:07:42.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:42.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:42.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:42.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:42.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:42.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:42.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:42.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:42.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:42.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:42.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:42.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:42.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:42.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:42.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:42.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:42.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:42.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:42.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:42.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:42.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:42.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:42.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:42.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:42.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:42.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:42.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:42.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:42.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:42.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:42.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:42.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:42.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:42.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:42.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:42.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:42.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:42.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:42.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:42.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:42.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:42.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=40, tag_tid=2, entry_tid=44652], [object_number=43, tag_tid=2, entry_tid=44651], [object_number=42, tag_tid=2, entry_tid=44650], [object_number=41, tag_tid=2, entry_tid=44649]]], state=connected] 2026-02-20T21:07:42.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:42.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:42.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:42.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=40, tag_tid=2, entry_tid=44652], [object_number=43, tag_tid=2, entry_tid=44651], [object_number=42, tag_tid=2, entry_tid=44650], [object_number=41, tag_tid=2, entry_tid=44649]]], state=connected] 2026-02-20T21:07:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=2, entry_tid=44652' 2026-02-20T21:07:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=40, tag_tid=2, entry_tid=44652' 2026-02-20T21:07:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=2, entry_tid=44652' '!=' 'object_number=43, tag_tid=2, entry_tid=44159' 2026-02-20T21:07:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=2, entry_tid=44652' = 'object_number=43, tag_tid=2, entry_tid=44159' 2026-02-20T21:07:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=40, tag_tid=2, entry_tid=44652' 2026-02-20T21:07:42.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:42.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:42.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:42.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:42.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=2, entry_tid=44652' 2026-02-20T21:07:42.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:42.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:42.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:42.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:42.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:42.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=2, entry_tid=44652' 2026-02-20T21:07:42.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:42.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44652 2026-02-20T21:07:42.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 44652 2026-02-20T21:07:42.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:42.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:42.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:42.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:42.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:42.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:42.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:42.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:42.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:42.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:42.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:42.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:42.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:42.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:42.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:42.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:42.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:42.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:42.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:42.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:42.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:42.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:42.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:42.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:42.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:42.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:43.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:43.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:43.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:43.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:43.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:43.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:43.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:43.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:43.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:43.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:43.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:43.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:43.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=45541], [object_number=40, tag_tid=2, entry_tid=45540], [object_number=43, tag_tid=2, entry_tid=45539], [object_number=42, tag_tid=2, entry_tid=45538]]], state=connected] 2026-02-20T21:07:43.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:43.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:43.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:43.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=45541], [object_number=40, tag_tid=2, entry_tid=45540], [object_number=43, tag_tid=2, entry_tid=45539], [object_number=42, tag_tid=2, entry_tid=45538]]], state=connected] 2026-02-20T21:07:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=45541' 2026-02-20T21:07:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=45541' 2026-02-20T21:07:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=2, entry_tid=45541' '!=' 'object_number=40, tag_tid=2, entry_tid=44652' 2026-02-20T21:07:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=2, entry_tid=45541' = 'object_number=40, tag_tid=2, entry_tid=44652' 2026-02-20T21:07:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=41, tag_tid=2, entry_tid=45541' 2026-02-20T21:07:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=45541' 2026-02-20T21:07:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=45541' 2026-02-20T21:07:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45541 2026-02-20T21:07:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 45541 2026-02-20T21:07:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:43.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:43.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:43.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:43.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:43.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:43.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:43.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:43.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:43.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:43.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:43.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:43.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:43.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:43.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:43.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:43.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:43.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:43.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:43.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:43.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:43.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:43.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:43.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:43.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:43.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:43.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:43.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:43.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:43.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:43.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:43.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:43.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:43.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:43.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:43.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:43.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:43.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:43.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:43.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:43.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=2, entry_tid=46118], [object_number=41, tag_tid=2, entry_tid=46117], [object_number=40, tag_tid=2, entry_tid=46116], [object_number=43, tag_tid=2, entry_tid=46115]]], state=connected] 2026-02-20T21:07:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:43.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:43.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:43.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:43.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:07:43.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:43.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:43.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:43.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=2, entry_tid=46118], [object_number=41, tag_tid=2, entry_tid=46117], [object_number=40, tag_tid=2, entry_tid=46116], [object_number=43, tag_tid=2, entry_tid=46115]]], state=connected] 2026-02-20T21:07:43.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:43.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=2, entry_tid=46118' 2026-02-20T21:07:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=42, tag_tid=2, entry_tid=46118' 2026-02-20T21:07:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=2, entry_tid=46118' '!=' 'object_number=41, tag_tid=2, entry_tid=45541' 2026-02-20T21:07:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=2, entry_tid=46118' = 'object_number=41, tag_tid=2, entry_tid=45541' 2026-02-20T21:07:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=42, tag_tid=2, entry_tid=46118' 2026-02-20T21:07:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:43.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:43.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=2, entry_tid=46118' 2026-02-20T21:07:43.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:43.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:43.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:43.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:43.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:43.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:43.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=2, entry_tid=46118' 2026-02-20T21:07:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:43.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46118 2026-02-20T21:07:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 46118 2026-02-20T21:07:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:43.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:43.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:43.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:43.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:43.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:43.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:43.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:43.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:43.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:43.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:43.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:43.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:43.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:43.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:44.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:44.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:44.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:44.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:44.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:44.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:44.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:44.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:44.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:44.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:44.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:44.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:44.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46661], [object_number=44, tag_tid=2, entry_tid=46660], [object_number=47, tag_tid=2, entry_tid=46659], [object_number=46, tag_tid=2, entry_tid=46658]]], state=connected] 2026-02-20T21:07:44.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:44.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:44.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46661], [object_number=44, tag_tid=2, entry_tid=46660], [object_number=47, tag_tid=2, entry_tid=46659], [object_number=46, tag_tid=2, entry_tid=46658]]], state=connected] 2026-02-20T21:07:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=46661' 2026-02-20T21:07:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=46661' 2026-02-20T21:07:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=46661' '!=' 'object_number=42, tag_tid=2, entry_tid=46118' 2026-02-20T21:07:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=46661' = 'object_number=42, tag_tid=2, entry_tid=46118' 2026-02-20T21:07:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=45, tag_tid=2, entry_tid=46661' 2026-02-20T21:07:44.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:44.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=46661' 2026-02-20T21:07:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:44.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:44.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:44.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:44.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=46661' 2026-02-20T21:07:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:44.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:44.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46661 2026-02-20T21:07:44.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 46661 2026-02-20T21:07:44.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:44.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:44.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:44.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:44.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:44.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:44.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:44.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:44.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:44.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:44.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:44.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:44.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:44.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:44.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:44.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:44.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:44.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:44.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:44.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:44.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:44.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:44.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:44.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:44.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:44.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:44.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:44.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:44.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:44.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:44.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:44.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:44.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:44.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:44.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:44.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:44.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:44.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:44.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:44.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:44.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:44.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:44.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:44.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:44.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:44.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:44.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:44.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:44.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46845], [object_number=44, tag_tid=2, entry_tid=46844], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:44.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:44.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:44.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:44.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:44.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:44.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:44.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:44.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:44.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:44.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:44.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:44.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:44.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46845], [object_number=44, tag_tid=2, entry_tid=46844], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:44.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=46845' '!=' 'object_number=45, tag_tid=2, entry_tid=46661' 2026-02-20T21:07:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:07:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=46845' = 'object_number=45, tag_tid=2, entry_tid=46661' 2026-02-20T21:07:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:07:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:07:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:07:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:07:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:07:44.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:07:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46845 2026-02-20T21:07:44.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 46845 2026-02-20T21:07:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:07:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:44.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:07:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:44.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:44.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:44.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:44.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:44.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:44.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:44.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:44.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:45.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:45.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:45.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:45.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:45.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:45.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:45.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:45.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:45.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:45.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46845], [object_number=44, tag_tid=2, entry_tid=46844], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:45.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:45.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:45.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:45.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:45.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:45.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:45.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46845], [object_number=44, tag_tid=2, entry_tid=46844], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=46845' '!=' 'object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:07:45.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:45.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:45.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:45.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:45.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:45.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:45.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:45.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:45.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:45.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:45.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:45.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:45.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:45.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:45.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:45.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:45.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:45.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:45.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:45.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:45.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:45.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:45.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:45.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:45.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:45.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:45.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:45.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:45.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:45.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:45.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:45.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:45.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:45.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:45.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:45.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:45.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:45.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:45.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:45.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:45.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:45.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:45.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:45.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:45.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46845], [object_number=44, tag_tid=2, entry_tid=46844], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:45.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:45.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:45.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:45.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:45.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:45.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:45.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:45.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:45.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46845], [object_number=44, tag_tid=2, entry_tid=46844], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:45.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:45.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:45.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=46845' '!=' 'object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:45.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:45.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:07:46.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:46.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:46.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:46.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:46.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:46.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:46.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:46.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:46.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:46.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:46.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:46.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:46.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:46.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:46.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:46.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:46.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:46.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:46.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:46.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:46.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:46.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:46.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:46.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:46.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:46.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:46.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:46.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:46.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:46.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:46.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:46.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:46.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46845], [object_number=44, tag_tid=2, entry_tid=46844], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:47.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46845], [object_number=44, tag_tid=2, entry_tid=46844], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=46845' '!=' 'object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:07:48.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:48.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:48.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:48.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:48.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:48.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:48.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:48.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:48.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:48.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:48.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:48.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:48.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:48.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:48.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:48.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:48.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:48.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:48.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:48.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:48.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:48.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:48.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:48.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:48.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:48.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:48.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:48.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:48.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:48.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:48.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:48.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:48.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:48.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46845], [object_number=44, tag_tid=2, entry_tid=46844], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:48.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:48.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:48.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:48.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:48.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:48.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:48.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:48.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:48.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=46845], [object_number=44, tag_tid=2, entry_tid=46844], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:48.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=2, entry_tid=46845' '!=' 'object_number=45, tag_tid=2, entry_tid=46845' 2026-02-20T21:07:48.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:07:48.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:07:50.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:07:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:07:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:07:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:07:50.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:07:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:07:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:07:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:07:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:07:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:07:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:07:50.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:07:50.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:07:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:07:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:07:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:07:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:07:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:07:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:07:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:07:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:07:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:07:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:07:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:07:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:07:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:07:50.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:07:50.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:50.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:50.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:07:50.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:07:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:50.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:50.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:07:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:07:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:07:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:07:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:50.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:07:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:07:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:07:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:07:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:50.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:07:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:07:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:07:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:07:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:07:50.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:07:50.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:07:50.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:50.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:50.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:07:50.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:07:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:07:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:07:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:07:51.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:07:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:07:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:07:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:07:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:07:51.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:07:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:07:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:07:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:07:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:07:51.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:07:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:07:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:51.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:51.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:07:51.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:07:51.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:07:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:07:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:07:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:07:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:07:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:07:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:07:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:07:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:07:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:07:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:07:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:07:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:07:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:07:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:07:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:07:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:07:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:07:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:07:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:07:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:07:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:07:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:07:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:07:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:07:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:07:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:07:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:07:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:07:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:07:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:07:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:07:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:07:51.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:07:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:51.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:07:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:07:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:07:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:51.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:51.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:51.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:51.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:51.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:51.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:51.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:07:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:07:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:07:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:07:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:07:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:07:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:07:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:07:51.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:07:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:51.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:07:51.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:07:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:07:51.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:07:51.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:51.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=3, entry_tid=1], [object_number=44, tag_tid=3, entry_tid=0], [object_number=47, tag_tid=2, entry_tid=46843], [object_number=46, tag_tid=2, entry_tid=46842]]], state=connected] 2026-02-20T21:07:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:07:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=3, entry_tid=1' -a 'object_number=45, tag_tid=3, entry_tid=1' = 'object_number=45, tag_tid=3, entry_tid=1' 2026-02-20T21:07:51.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap1 2026-02-20T21:07:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:07:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1 2026-02-20T21:07:51.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:07:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:07:51.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:07:52.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap1 2026-02-20T21:07:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-20T21:07:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:07:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:07:52.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ba958b175d 2026-02-20T21:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ba958b175d 2026-02-20T21:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:06:40 2026 2026-02-20T21:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:06:40 2026 2026-02-20T21:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:07:41 2026 2026-02-20T21:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:07:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ba958b175d 2026-02-20T21:07:52.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:07:52.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:07:52.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:07:52.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:07:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:07:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap1 2026-02-20T21:07:52.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1 2026-02-20T21:07:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:07:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:07:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:07:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:07:52.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:07:52.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:07:52.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:07:52.584 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:07:52.582+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:07:52.585 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:07:52.583+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:07:52.586 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:07:52.583+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:07:52.586 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:07:52.584+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:07:52.586 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:07:52.584+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:07:52.586 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:07:52.584+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:07:52.799 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:07:52.798+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:07:52.800 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:07:52.798+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:07:52.801 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:07:52.799+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:07:52.801 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:07:52.799+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:07:52.801 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:07:52.799+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:07:52.801 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:07:52.799+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:07:52.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:07:52.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:07:53.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:07:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:07:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap1-clean 2026-02-20T21:07:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap1-clean 2026-02-20T21:07:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:07:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1-clean 2026-02-20T21:07:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap1-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:07:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:07:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:07:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:07:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:07:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:07:53.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:07:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-20T21:07:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:07:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:07:54.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 3468288 bytes 2026-02-20T21:07:54.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 46024704 bytes 2026-02-20T21:07:55.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 77286400 bytes 2026-02-20T21:07:56.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 145496064 bytes 2026-02-20T21:07:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 182446080 bytes 2026-02-20T21:07:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 191420416 bytes 2026-02-20T21:07:57.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 279782400 bytes 2026-02-20T21:07:58.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 350178304 bytes 2026-02-20T21:07:59.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 423088128 bytes 2026-02-20T21:08:00.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 515865600 bytes 2026-02-20T21:08:00.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 519092224 bytes 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:08:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:08:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:08:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:08:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:08:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:08:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:08:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:08:01.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:08:01.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:08:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:08:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:08:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-20T21:08:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":18098956.47,"entries_behind_primary":5966,"entries_per_second":1129.53,"non_primary_position":{"entry_tid":40880,"object_number":36,"tag_tid":2},"primary_position":{"entry_tid":1,"object_number":45,"tag_tid":3},"seconds_until_synced":5} 2026-02-20T21:08:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.3 on trial163 2026-02-20T21:08:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:07:38 2026-02-20T21:08:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:08:01.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:08:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:08:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:08:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:07:38' 2026-02-20T21:08:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:08:01.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-20T21:08:01.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-20T21:08:01.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":18098956.47,"entries_behind_primary":5966,"entries_per_second":1129.53,"non_primary_position":{"entry_tid":40880,"object_number":36,"tag_tid":2},"primary_position":{"entry_tid":1,"object_number":45,"tag_tid":3},"seconds_until_synced":5} 2026-02-20T21:08:01.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:08:01.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:08:01.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-20T21:08:01.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:08:01.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:08:01.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.3 on trial163 2026-02-20T21:08:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-20T21:08:01.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:08:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:07:38' 2026-02-20T21:08:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:07:38' 2026-02-20T21:08:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:08:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:08:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:08:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:08:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-20 21:07:38' 2026-02-20T21:08:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:08:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:08:01.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:08:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142044287mirror.0trial16320.2.0-677-gf7870454trueOK42084286mirror.1trial16320.2.0-677-gf7870454falseOK42164285mirror.3trial16320.2.0-677-gf7870454falseOK43304332mirror.2trial16320.2.0-677-gf7870454falseOKtestba923525-490d-4a76-8e5f-b742509a94daup+replayingreplaying, {"bytes_per_second":18098956.47,"entries_behind_primary":5966,"entries_per_second":1129.53,"non_primary_position":{"entry_tid":40880,"object_number":36,"tag_tid":2},"primary_position":{"entry_tid":1,"object_number":45,"tag_tid":3},"seconds_until_synced":5}42164285mirror.3trial1632026-02-20 21:07:38cluster28b968eaf-6c24-469f-bea5-c8acb780219eup+stoppedlocal image is primary2026-02-20 21:07:38' 2026-02-20T21:08:01.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-20T21:08:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:08:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:07:38' 2026-02-20T21:08:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:08:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-20T21:08:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:08:01.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-20T21:08:01.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:08:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:08:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:08:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap2 2026-02-20T21:08:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap2 2026-02-20T21:08:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:08:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:08:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:08:02.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2 2026-02-20T21:08:02.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:08:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:08:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:08:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:08:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:08:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:08:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:08:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:08:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:08:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:08:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:08:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:08:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:08:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:08:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:08:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:08:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:08:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:08:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:08:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:08:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:08:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:08:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:08:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:08:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:08:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:32.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:08:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:08:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:08:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:08:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:08:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:08:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:08:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:08:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:08:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:08:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:08:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:08:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:08:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:08:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:08:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:08:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:08:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:08:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:32.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:08:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:08:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:08:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:08:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:08:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:08:32.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:08:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:08:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:08:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:08:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:08:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:08:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:08:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:08:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:08:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:08:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:08:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:08:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:08:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:08:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:08:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:08:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:08:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:08:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:08:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:08:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:08:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:08:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:08:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:08:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:08:33.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:33.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:33.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:33.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:08:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:08:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:08:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:08:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:08:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:08:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:08:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:33.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:08:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:08:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:08:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:08:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:08:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:08:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:08:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:08:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:08:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:08:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:33.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:08:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:08:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:08:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:08:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:08:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:08:33.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:08:33.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:33.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:33.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:33.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:08:33.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:33.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:08:33.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:33.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:33.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-20T21:08:33.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:08:33.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:08:33.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:33.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:08:33.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:08:33.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:08:33.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:33.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:33.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:33.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:33.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:08:33.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:08:33.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:08:33.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:08:33.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:08:33.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:08:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:08:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:08:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:08:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:08:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:08:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:08:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:08:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:08:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:08:33.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:08:33.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:08:33.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:08:33.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:08:33.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:33.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:33.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:33.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-20T21:08:33.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:08:33.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:08:33.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:08:33.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:08:33.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:08:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:08:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:08:33.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:08:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:08:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:08:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:08:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:08:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:08:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:08:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:08:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:08:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:08:33.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:08:33.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:08:33.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:08:33.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:08:33.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:08:33.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:08:33.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:08:33.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:08:33.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:08:33.315 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.3362576726603239) > weight (0.25) 2026-02-20T21:08:33.315 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.8873698263524192) > weight (0.25) 2026-02-20T21:08:33.316 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.6687393479618902) > weight (0.25) 2026-02-20T21:08:33.316 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-20T21:08:33.316 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-20T21:08:33.316 INFO:tasks.rbd_mirror_thrash:waiting for 0.6690027363555895 secs before reviving daemons 2026-02-20T21:08:33.317 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:08:33.315+0000 7f456a851640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 47188) UID: 1000 2026-02-20T21:08:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:08:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:08:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:08:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:08:33.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:08:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:08:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:08:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:08:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:08:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:08:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:08:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:08:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:08:33.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:08:33.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:08:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:08:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:08:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:08:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:08:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:08:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:08:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:08:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:08:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:08:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:08:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:33.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:33.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:33.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:33.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:33.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:08:33.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:08:33.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:08:33.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:33.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:33.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:33.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:08:33.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:33.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:33.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:33.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:33.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:08:33.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:08:33.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:33.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:33.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:08:33.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:08:33.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:33.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:33.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:33.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:08:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:08:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:08:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:08:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:08:33.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:08:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:08:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:08:33.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:08:33.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:33.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:33.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:33.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:08:33.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:33.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:33.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:33.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:08:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:08:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:08:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:08:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:33.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:08:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:08:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:08:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:08:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:08:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:08:33.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:08:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:08:33.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:08:33.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:33.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:33.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:33.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:08:33.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:33.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:33.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:08:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:08:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:33.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:08:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:08:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:08:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:08:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:08:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:08:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:08:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:08:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:33.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:08:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:08:33.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:08:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:33.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:33.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:08:33.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-20T21:08:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:08:33.986 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-20T21:08:33.986 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:08:33.987 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-20T21:08:33.987 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-20T21:08:33.987 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-20T21:08:33.990 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-20T21:08:33.990 INFO:tasks.rbd_mirror_thrash:waiting for 118.86608635163371 secs before thrashing 2026-02-20T21:08:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:08:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:08:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:08:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:08:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:08:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:08:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:34.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:08:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:08:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:08:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:08:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:08:34.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:08:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:08:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:08:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:08:34.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:08:34.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:34.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:34.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:34.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:34.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:08:34.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:08:34.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:34.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:34.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:08:34.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:08:34.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:34.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:08:34.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:08:34.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:08:34.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:34.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:34.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:34.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:08:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:08:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:08:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:08:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:34.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:08:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:08:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:08:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:08:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:08:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:08:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:08:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:08:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:08:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:08:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:34.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:34.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:34.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:08:34.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:08:34.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:08:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:08:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:34.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:08:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:08:34.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:34.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:08:34.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:08:34.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:34.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:34.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:08:34.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:08:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:34.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:08:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:08:34.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:08:34.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:34.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:34.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:34.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:08:34.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:08:35.631 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5557624505555594) > weight (0.25) 2026-02-20T21:08:35.631 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3241710779703979) > weight (0.25) 2026-02-20T21:08:35.631 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.7757775405997172) > weight (0.25) 2026-02-20T21:08:35.631 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.37026737611721794) > weight (0.25) 2026-02-20T21:08:35.631 INFO:tasks.rbd_mirror_thrash:waiting for 113.47859330848374 secs before thrashing 2026-02-20T21:08:36.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:08:36.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:08:36.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:36.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:08:36.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:08:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:08:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:08:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:08:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:08:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:08:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:08:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:08:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:08:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:08:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:08:36.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:08:36.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:36.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:36.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:36.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:08:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:36.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:36.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:08:36.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:08:36.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:36.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:08:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:08:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:08:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:08:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:08:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:08:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:08:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:08:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:36.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:08:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:08:36.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:08:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:08:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:08:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:08:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:08:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:08:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:08:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:08:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:08:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:08:36.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:08:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:08:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:36.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:36.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:08:36.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:08:36.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:08:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:36.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:08:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:08:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:08:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:08:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:08:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:08:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:08:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:08:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:08:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:08:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:08:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:36.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:36.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:36.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:36.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:08:36.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:08:36.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:08:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:08:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-20T21:08:41.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:08:41.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:08:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:08:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:08:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:08:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:08:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:08:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:08:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:08:41.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:08:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:08:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:08:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:08:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:08:41.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:08:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:41.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:08:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:41.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:41.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:08:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:08:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:08:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:08:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:08:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:08:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:08:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:08:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:08:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:08:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:41.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:41.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:41.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:41.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:41.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:41.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:41.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:08:41.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:08:41.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:08:41.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:41.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:41.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:41.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:08:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:41.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:08:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:08:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:08:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:08:41.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:08:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:08:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:08:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:08:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:41.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:08:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:08:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:08:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:08:41.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:08:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:41.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:08:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:08:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:08:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:08:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:08:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:08:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:08:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:08:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:08:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:08:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:08:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:08:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:08:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:08:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:41.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:08:41.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:41.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:41.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:41.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 8 2026-02-20T21:08:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:08:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:08:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:08:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:08:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:08:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:08:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:08:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:08:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:08:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:08:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:08:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:08:49.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:49.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:49.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:49.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:49.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:49.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:49.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:49.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:08:49.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:08:49.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:08:49.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:49.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:49.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:49.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:08:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:49.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:08:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:08:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:08:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:08:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:08:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:08:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:08:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:08:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:08:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:08:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:49.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:08:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:08:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:08:49.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:08:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:49.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:08:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:08:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:08:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:08:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:49.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:08:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:08:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:08:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:08:49.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:08:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:08:49.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:49.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:49.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:08:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:08:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:08:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:08:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:08:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:08:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:08:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:08:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:08:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:08:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:08:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:08:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:08:49.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:49.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:08:49.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:49.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:49.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:49.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:49.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 8 2026-02-20T21:08:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:08:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:08:57.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:57.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:08:57.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:08:57.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:08:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:08:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:08:57.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:08:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:08:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:08:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:08:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:08:57.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:08:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:08:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:08:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:08:57.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:08:57.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:57.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:57.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:08:57.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:08:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:57.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:57.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:57.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:08:57.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:08:57.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:57.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:57.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:08:57.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:08:57.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:57.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:57.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:57.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:08:57.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:57.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:57.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:57.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:08:57.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:57.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:08:57.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:08:57.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:08:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:08:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:08:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:08:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:57.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:08:57.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:08:57.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:08:57.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:08:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:08:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:08:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:08:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:08:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:08:57.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:08:57.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:57.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:57.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/test 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:08:57.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:08:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:08:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:08:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:08:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:08:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:08:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:08:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:08:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:08:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:08:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:08:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:08:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/test 2026-02-20T21:08:57.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:08:57.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:08:57.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:08:57.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:08:57.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 8 2026-02-20T21:09:05.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:09:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:05.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:05.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:05.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:05.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:09:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:09:05.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:09:05.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:09:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:09:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:09:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:09:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:09:05.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:09:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:09:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:09:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:09:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:09:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:09:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:09:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:09:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:09:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:09:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:09:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:09:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:05.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:05.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=77, tag_tid=6, entry_tid=1' 2026-02-20T21:09:05.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:09:05.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:09:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:09:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:05.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:05.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:05.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:05.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:05.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=77, tag_tid=6, entry_tid=1' -a 'object_number=77, tag_tid=6, entry_tid=1' = 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:05.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=33558' '!=' '' 2026-02-20T21:09:05.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:09:05.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=33558' = '' 2026-02-20T21:09:05.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:05.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=77, tag_tid=6, entry_tid=1' 2026-02-20T21:09:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:09:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:09:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=6 2026-02-20T21:09:05.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:05.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:09:05.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:09:05.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=5 2026-02-20T21:09:05.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=77, tag_tid=6, entry_tid=1' 2026-02-20T21:09:05.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:09:05.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:09:05.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:09:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:09:05.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:09:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33558 2026-02-20T21:09:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 6 = 5 -a 1 -le 33558 2026-02-20T21:09:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:09:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:09:05.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:09:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:09:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:09:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:09:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:09:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:09:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:09:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:09:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:09:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:06.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:09:06.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:09:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:09:06.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:09:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:09:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:09:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:06.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:09:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:09:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:09:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:06.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:06.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:06.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:06.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:06.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:06.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:06.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:09:06.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:09:06.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:09:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:09:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:09:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:09:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:09:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:09:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:09:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:09:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:09:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:09:06.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:06.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:06.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:06.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:09:06.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:09:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:09:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:06.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:06.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:06.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:09:06.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:06.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:06.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:06.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:06.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:06.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:06.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:06.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:06.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:06.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:06.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=77, tag_tid=6, entry_tid=1' 2026-02-20T21:09:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:09:06.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:06.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:06.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:06.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:06.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:06.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:06.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=77, tag_tid=6, entry_tid=1' -a 'object_number=77, tag_tid=6, entry_tid=1' = 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:06.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=33558' '!=' 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:06.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:09:06.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:09:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:09:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:09:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:09:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:09:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:09:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:09:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:09:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:09:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:09:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:06.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:09:06.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:09:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:09:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:09:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:09:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:09:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:09:06.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:06.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:06.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:06.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:09:06.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:06.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:06.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:06.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:06.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:06.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:09:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:09:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:09:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:09:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:09:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:06.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:09:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:09:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:09:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:09:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:09:06.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:09:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:09:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:09:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:09:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:09:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:06.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:09:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:09:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:09:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:09:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:09:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:09:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:06.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:09:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:06.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:06.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:06.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:06.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=77, tag_tid=6, entry_tid=1' 2026-02-20T21:09:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:09:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:09:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:09:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:06.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:06.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:06.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:06.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:06.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:06.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:06.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=77, tag_tid=6, entry_tid=1' -a 'object_number=77, tag_tid=6, entry_tid=1' = 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=33558' '!=' 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:09:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:09:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:09:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:09:07.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:09:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:09:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:09:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:09:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:09:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:09:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:09:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:09:07.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:07.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:09:07.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:09:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:09:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:09:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:09:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:07.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:09:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:09:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:07.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:07.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:07.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:09:07.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:09:07.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:07.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:07.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:07.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:07.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:09:07.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:07.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:09:07.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:09:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:09:07.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:09:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:09:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:09:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:09:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:07.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:09:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:09:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:09:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:09:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:09:07.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:09:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:09:07.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:09:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:07.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:07.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:09:07.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:09:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:09:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:07.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:09:07.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:07.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:07.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:07.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:07.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:07.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:07.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=77, tag_tid=6, entry_tid=1' 2026-02-20T21:09:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:09:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:09:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:09:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:07.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=77, tag_tid=6, entry_tid=1' -a 'object_number=77, tag_tid=6, entry_tid=1' = 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=33558' '!=' 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:09:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:09:09.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:09:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:09:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:09:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:09:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:09:09.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:09:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:09:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:09:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:09:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:09.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:09:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:09:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:09:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:09:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:09:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:09:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:09:09.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:09.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:09.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:09.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:09:09.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:09.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:09.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:09.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:09.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:09.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:09.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:09.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:09:09.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:09:09.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:09.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:09.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:09.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:09.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:09:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:09.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:09:09.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:09:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:09:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:09:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:09:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:09:09.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:09:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:09:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:09:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:09:09.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:09:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:09:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:09:09.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:09:09.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:09:09.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:09.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:09.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:09.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:09:09.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:09:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:09:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:09:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:09.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:09.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:09.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:09.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:09.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:09.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:09.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:09.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=77, tag_tid=6, entry_tid=1' 2026-02-20T21:09:09.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:09:09.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:09.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:09.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:09.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:09:09.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:09.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:09.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:09.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:09:09.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:09.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:09.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:09.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:09.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:09.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:09.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:09.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=77, tag_tid=6, entry_tid=1' -a 'object_number=77, tag_tid=6, entry_tid=1' = 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=33558' '!=' 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:09:09.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:09:11.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:09:11.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:11.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:11.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:11.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:09:11.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:09:11.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:09:11.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:09:11.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:09:11.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:11.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:11.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:09:11.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:09:11.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:11.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:09:11.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:09:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:09:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:09:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:09:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:11.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:09:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:09:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:09:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:11.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:09:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:09:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:09:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:11.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:11.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:11.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:11.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:09:11.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:09:11.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:09:11.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:11.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:09:11.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:09:11.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:11.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:09:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:09:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:09:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:09:11.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:09:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:09:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:09:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:09:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:11.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:09:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:09:11.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:09:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:09:11.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:09:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:09:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:11.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:09:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:11.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:11.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:11.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=77, tag_tid=6, entry_tid=1' 2026-02-20T21:09:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:09:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:09:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:09:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:11.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:11.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:11.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:11.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=77, tag_tid=6, entry_tid=1' -a 'object_number=77, tag_tid=6, entry_tid=1' = 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:11.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=33558' '!=' 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:11.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:09:11.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:09:13.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:09:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:09:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:09:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:09:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:09:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:09:13.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:09:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:09:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:09:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:09:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:09:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:09:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:09:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:13.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:09:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:09:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:09:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:09:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:09:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:09:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:13.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:13.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:13.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:13.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:09:13.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:09:13.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:09:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:09:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:09:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:09:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:09:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:09:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:09:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:09:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:09:13.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:09:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:09:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:09:13.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:09:13.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:09:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:13.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:13.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:09:13.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:09:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:09:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:09:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:13.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=77, tag_tid=6, entry_tid=1' 2026-02-20T21:09:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:09:13.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:09:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:09:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:13.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:13.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=78, tag_tid=5, entry_tid=33558], [object_number=77, tag_tid=5, entry_tid=33557], [object_number=76, tag_tid=5, entry_tid=33556], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:13.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=77, tag_tid=6, entry_tid=1' -a 'object_number=77, tag_tid=6, entry_tid=1' = 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=78, tag_tid=5, entry_tid=33558' '!=' 'object_number=78, tag_tid=5, entry_tid=33558' 2026-02-20T21:09:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:09:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-20T21:09:17.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:09:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:09:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:09:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:09:17.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:09:17.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:09:17.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:17.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:17.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:09:17.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:09:17.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:17.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:09:17.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:09:17.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:09:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:09:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:09:17.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:09:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:09:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:09:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:17.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:09:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:09:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:09:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:09:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:09:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:17.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:09:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:09:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:09:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:09:17.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:09:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:09:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:09:18.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:09:18.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:09:18.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:09:18.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:09:18.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:09:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:09:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:09:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:09:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:09:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:09:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:09:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:09:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:09:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:09:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:09:18.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:09:18.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:09:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:09:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:09:18.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:09:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:09:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:09:18.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:18.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:18.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:18.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:18.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=77, tag_tid=6, entry_tid=1' 2026-02-20T21:09:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:09:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:09:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:09:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:09:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:09:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:09:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:09:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:09:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:09:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:09:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 19 2026-02-20T21:09:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 19 2026-02-20T21:09:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:09:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=77, tag_tid=6, entry_tid=1], [object_number=76, tag_tid=6, entry_tid=0], [object_number=78, tag_tid=5, entry_tid=33558], [object_number=79, tag_tid=5, entry_tid=33555]]], state=connected] 2026-02-20T21:09:18.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:09:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=77, tag_tid=6, entry_tid=1' 2026-02-20T21:09:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=77, tag_tid=6, entry_tid=1' -a 'object_number=77, tag_tid=6, entry_tid=1' = 'object_number=77, tag_tid=6, entry_tid=1' 2026-02-20T21:09:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap2 2026-02-20T21:09:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2 2026-02-20T21:09:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:09:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:09:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:09:19.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap2 2026-02-20T21:09:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-20T21:09:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:09:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:09:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 3 2026-02-20T21:09:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ba958b175d 2026-02-20T21:09:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ba958b175d 2026-02-20T21:09:19.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:09:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:09:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:09:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:09:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:06:40 2026 2026-02-20T21:09:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:07:52 2026 2026-02-20T21:09:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:07:41 2026 2026-02-20T21:09:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:09:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ba958b175d 2026-02-20T21:09:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:09:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:09:19.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:09:19.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:09:19.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:19.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap1-clean ']' 2026-02-20T21:09:19.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap1-clean 2026-02-20T21:09:19.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:19.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1-clean 2026-02-20T21:09:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:09:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:09:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:09:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:09:19.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:09:19.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:09:19.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:09:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:09:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:09:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:09:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap2 2026-02-20T21:09:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2 2026-02-20T21:09:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:09:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:09:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:09:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:09:20.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:09:20.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:09:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:09:20.608 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.606+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:09:20.608 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.606+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:09:20.611 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.609+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:09:20.611 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.609+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:09:20.611 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.609+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:09:20.611 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.609+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:09:20.689 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.687+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:09:20.753 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.751+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:09:20.754 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.752+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:09:20.754 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.753+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:09:20.755 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.753+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:09:20.755 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.754+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:09:20.788 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.786+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:09:20.788 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.786+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:09:20.858 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.857+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:09:20.859 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.858+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:09:20.859 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.858+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:09:20.860 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.858+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:09:20.860 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.858+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:09:20.860 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.859+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:09:20.861 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.860+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:09:20.894 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.893+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:09:20.895 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.893+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:09:20.895 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.893+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:09:20.896 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.894+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:09:20.896 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.894+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:09:20.896 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:09:20.894+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:09:21.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:09:21.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:09:21.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:09:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap2-clean 2026-02-20T21:09:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap2-clean 2026-02-20T21:09:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:09:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2-clean 2026-02-20T21:09:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap2-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:09:21.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-20T21:09:21.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:09:21.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:09:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 32407552 bytes 2026-02-20T21:09:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 58074112 bytes 2026-02-20T21:09:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 108137472 bytes 2026-02-20T21:09:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 132381696 bytes 2026-02-20T21:09:23.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 133009408 bytes 2026-02-20T21:09:23.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 180272128 bytes 2026-02-20T21:09:24.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 219684864 bytes 2026-02-20T21:09:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 270132224 bytes 2026-02-20T21:09:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 315168768 bytes 2026-02-20T21:09:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 372596736 bytes 2026-02-20T21:09:27.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 401574912 bytes 2026-02-20T21:09:27.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 439403520 bytes 2026-02-20T21:09:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 469166080 bytes 2026-02-20T21:09:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 491013120 bytes 2026-02-20T21:09:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 594032640 bytes 2026-02-20T21:09:29.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 613027840 bytes 2026-02-20T21:09:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 678235136 bytes 2026-02-20T21:09:30.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 694283264 bytes 2026-02-20T21:09:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 713358336 bytes 2026-02-20T21:09:31.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 792759296 bytes 2026-02-20T21:09:32.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 851634176 bytes 2026-02-20T21:09:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2200 IOs, 921494528 bytes 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:09:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:09:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:09:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:09:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:09:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:09:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:09:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:09:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:09:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:09:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-20T21:09:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":33558,"object_number":78,"tag_tid":5},"primary_position":{"entry_tid":1,"object_number":77,"tag_tid":6}} 2026-02-20T21:09:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.1 on trial163 2026-02-20T21:09:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:09:08 2026-02-20T21:09:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:09:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:09:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:09:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:09:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:09:08' 2026-02-20T21:09:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:09:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-20T21:09:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-20T21:09:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":33558,"object_number":78,"tag_tid":5},"primary_position":{"entry_tid":1,"object_number":77,"tag_tid":6}} 2026-02-20T21:09:34.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:09:34.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:09:34.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-20T21:09:34.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:09:34.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:09:34.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.1 on trial163 2026-02-20T21:09:34.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-20T21:09:34.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:09:34.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:09:08' 2026-02-20T21:09:34.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:09:08' 2026-02-20T21:09:34.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:09:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:09:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-20 21:09:08' 2026-02-20T21:09:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:09:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:09:34.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:09:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142044287mirror.0trial16320.2.0-677-gf7870454trueOK42084286mirror.1trial16320.2.0-677-gf7870454falseOK43304332mirror.2trial16320.2.0-677-gf7870454falseOK43484351mirror.3trial16320.2.0-677-gf7870454falseOKtestba923525-490d-4a76-8e5f-b742509a94daup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":33558,"object_number":78,"tag_tid":5},"primary_position":{"entry_tid":1,"object_number":77,"tag_tid":6}}42084286mirror.1trial1632026-02-20 21:09:08cluster28b968eaf-6c24-469f-bea5-c8acb780219eup+stoppedlocal image is primary2026-02-20 21:09:08' 2026-02-20T21:09:34.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-20T21:09:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:09:34.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:09:08' 2026-02-20T21:09:34.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:09:34.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-20T21:09:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:09:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-20T21:09:34.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:09:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:09:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap3 2026-02-20T21:09:34.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap3 2026-02-20T21:09:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:09:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:09:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:09:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3 2026-02-20T21:09:34.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:10:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:10:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:10:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:10:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:10:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:10:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:10:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:06.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:10:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:10:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:10:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:10:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:10:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:10:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:10:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:10:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:10:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:06.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:10:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:10:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:10:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:10:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:10:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:10:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:10:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:10:06.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:10:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:10:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:10:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:10:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:10:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:10:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:10:06.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:10:06.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:10:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:10:06.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:06.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:10:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:10:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:10:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:10:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:06.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:10:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:10:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:10:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:10:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:10:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:10:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:10:06.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:10:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:10:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:10:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:10:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:10:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:10:06.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:06.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:06.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:06.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:06.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:10:06.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:10:06.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:10:06.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:10:06.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:10:06.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:10:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:10:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:10:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:10:06.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:10:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:10:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:10:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:10:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:10:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:10:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:10:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:10:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:10:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:10:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:06.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:10:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:10:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:07.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:07.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:07.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:07.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:07.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:07.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:07.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:07.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:07.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:07.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:07.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:07.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 28 2026-02-20T21:10:07.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:07.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:07.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:07.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=113, tag_tid=8, entry_tid=40185], [object_number=112, tag_tid=8, entry_tid=40184], [object_number=115, tag_tid=8, entry_tid=40183], [object_number=114, tag_tid=8, entry_tid=40182]]], state=connected] 2026-02-20T21:10:07.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:07.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:07.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:07.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:07.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:07.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 28 2026-02-20T21:10:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:07.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=113, tag_tid=8, entry_tid=40185], [object_number=112, tag_tid=8, entry_tid=40184], [object_number=115, tag_tid=8, entry_tid=40183], [object_number=114, tag_tid=8, entry_tid=40182]]], state=connected] 2026-02-20T21:10:07.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=113, tag_tid=8, entry_tid=40185' 2026-02-20T21:10:07.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=113, tag_tid=8, entry_tid=40185' 2026-02-20T21:10:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=113, tag_tid=8, entry_tid=40185' '!=' '' 2026-02-20T21:10:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=113, tag_tid=8, entry_tid=40185' = '' 2026-02-20T21:10:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=113, tag_tid=8, entry_tid=40185' 2026-02-20T21:10:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:07.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=113, tag_tid=8, entry_tid=40185' 2026-02-20T21:10:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:07.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:07.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=113, tag_tid=8, entry_tid=40185' 2026-02-20T21:10:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:07.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40185 2026-02-20T21:10:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 40185 2026-02-20T21:10:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:07.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:07.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:07.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:07.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:07.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:07.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:07.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:07.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:07.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:07.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:07.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:07.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:07.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:07.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:07.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:07.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:07.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:07.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:07.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:07.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:07.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:07.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:07.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:07.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:07.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:07.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:07.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:07.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:07.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:07.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:07.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:07.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:07.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:07.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:07.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:07.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:07.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:07.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:07.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:07.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:07.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:07.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:07.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:07.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:07.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:07.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:07.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:07.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:07.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:07.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:07.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:07.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:07.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:07.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:07.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:07.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:07.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:07.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:07.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:07.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:07.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:07.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=40486], [object_number=117, tag_tid=8, entry_tid=40485], [object_number=116, tag_tid=8, entry_tid=40484], [object_number=119, tag_tid=8, entry_tid=40483]]], state=connected] 2026-02-20T21:10:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:07.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:07.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:07.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:07.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=40486], [object_number=117, tag_tid=8, entry_tid=40485], [object_number=116, tag_tid=8, entry_tid=40484], [object_number=119, tag_tid=8, entry_tid=40483]]], state=connected] 2026-02-20T21:10:07.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=118, tag_tid=8, entry_tid=40486' 2026-02-20T21:10:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=118, tag_tid=8, entry_tid=40486' 2026-02-20T21:10:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=40486' '!=' 'object_number=113, tag_tid=8, entry_tid=40185' 2026-02-20T21:10:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=40486' = 'object_number=113, tag_tid=8, entry_tid=40185' 2026-02-20T21:10:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=118, tag_tid=8, entry_tid=40486' 2026-02-20T21:10:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=40486' 2026-02-20T21:10:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:07.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:07.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=40486' 2026-02-20T21:10:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40486 2026-02-20T21:10:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 40486 2026-02-20T21:10:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:07.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:07.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:07.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:07.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:07.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:07.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:07.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:07.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:07.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:07.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:07.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:07.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:07.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:07.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=40860], [object_number=119, tag_tid=8, entry_tid=40859], [object_number=118, tag_tid=8, entry_tid=40858], [object_number=117, tag_tid=8, entry_tid=40857]]], state=connected] 2026-02-20T21:10:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:07.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:08.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=40860], [object_number=119, tag_tid=8, entry_tid=40859], [object_number=118, tag_tid=8, entry_tid=40858], [object_number=117, tag_tid=8, entry_tid=40857]]], state=connected] 2026-02-20T21:10:08.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=116, tag_tid=8, entry_tid=40860' 2026-02-20T21:10:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=116, tag_tid=8, entry_tid=40860' 2026-02-20T21:10:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=40860' '!=' 'object_number=118, tag_tid=8, entry_tid=40486' 2026-02-20T21:10:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=40860' = 'object_number=118, tag_tid=8, entry_tid=40486' 2026-02-20T21:10:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=116, tag_tid=8, entry_tid=40860' 2026-02-20T21:10:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:08.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:08.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=40860' 2026-02-20T21:10:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:08.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:08.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:08.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:08.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=40860' 2026-02-20T21:10:08.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:08.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40860 2026-02-20T21:10:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 40860 2026-02-20T21:10:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:08.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:08.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:08.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:08.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:08.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:08.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:08.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:08.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:08.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:08.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:08.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:08.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:08.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=41229], [object_number=116, tag_tid=8, entry_tid=41228], [object_number=119, tag_tid=8, entry_tid=41227], [object_number=118, tag_tid=8, entry_tid=41226]]], state=connected] 2026-02-20T21:10:08.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:08.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:08.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:08.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:08.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:08.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:08.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:08.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:08.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:08.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:08.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:08.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:08.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:08.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:08.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=41229], [object_number=116, tag_tid=8, entry_tid=41228], [object_number=119, tag_tid=8, entry_tid=41227], [object_number=118, tag_tid=8, entry_tid=41226]]], state=connected] 2026-02-20T21:10:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=117, tag_tid=8, entry_tid=41229' 2026-02-20T21:10:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=117, tag_tid=8, entry_tid=41229' 2026-02-20T21:10:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=41229' '!=' 'object_number=116, tag_tid=8, entry_tid=40860' 2026-02-20T21:10:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=41229' = 'object_number=116, tag_tid=8, entry_tid=40860' 2026-02-20T21:10:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=117, tag_tid=8, entry_tid=41229' 2026-02-20T21:10:08.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:08.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:08.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:08.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:08.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=41229' 2026-02-20T21:10:08.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:08.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=41229' 2026-02-20T21:10:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:08.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41229 2026-02-20T21:10:08.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 41229 2026-02-20T21:10:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:08.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:08.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:08.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:08.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:08.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:08.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:08.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:08.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:08.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:08.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:08.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:08.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:08.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:08.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:08.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:08.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:08.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:08.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:08.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=41640], [object_number=119, tag_tid=8, entry_tid=41639], [object_number=118, tag_tid=8, entry_tid=41638], [object_number=117, tag_tid=8, entry_tid=41637]]], state=connected] 2026-02-20T21:10:08.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:08.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:08.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:08.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:08.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:08.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=41640], [object_number=119, tag_tid=8, entry_tid=41639], [object_number=118, tag_tid=8, entry_tid=41638], [object_number=117, tag_tid=8, entry_tid=41637]]], state=connected] 2026-02-20T21:10:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=116, tag_tid=8, entry_tid=41640' 2026-02-20T21:10:08.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=116, tag_tid=8, entry_tid=41640' 2026-02-20T21:10:08.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=41640' '!=' 'object_number=117, tag_tid=8, entry_tid=41229' 2026-02-20T21:10:08.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:08.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=41640' = 'object_number=117, tag_tid=8, entry_tid=41229' 2026-02-20T21:10:08.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=116, tag_tid=8, entry_tid=41640' 2026-02-20T21:10:08.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:08.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:08.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:08.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:08.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=41640' 2026-02-20T21:10:08.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:08.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:08.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:08.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:08.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:08.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:08.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:08.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=41640' 2026-02-20T21:10:08.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:08.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:08.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41640 2026-02-20T21:10:08.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 41640 2026-02-20T21:10:08.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:08.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:08.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:09.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:09.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:09.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:09.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:09.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=42014], [object_number=117, tag_tid=8, entry_tid=42013], [object_number=116, tag_tid=8, entry_tid=42012], [object_number=119, tag_tid=8, entry_tid=42011]]], state=connected] 2026-02-20T21:10:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=42014], [object_number=117, tag_tid=8, entry_tid=42013], [object_number=116, tag_tid=8, entry_tid=42012], [object_number=119, tag_tid=8, entry_tid=42011]]], state=connected] 2026-02-20T21:10:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=118, tag_tid=8, entry_tid=42014' 2026-02-20T21:10:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=118, tag_tid=8, entry_tid=42014' 2026-02-20T21:10:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=42014' '!=' 'object_number=116, tag_tid=8, entry_tid=41640' 2026-02-20T21:10:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=42014' = 'object_number=116, tag_tid=8, entry_tid=41640' 2026-02-20T21:10:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=118, tag_tid=8, entry_tid=42014' 2026-02-20T21:10:09.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:09.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=42014' 2026-02-20T21:10:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:09.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:09.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:09.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=42014' 2026-02-20T21:10:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:09.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42014 2026-02-20T21:10:09.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 42014 2026-02-20T21:10:09.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:09.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:09.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:09.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:09.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:09.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:09.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:09.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:09.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:09.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:09.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:09.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:09.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:09.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:09.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:09.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:09.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:09.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:09.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=119, tag_tid=8, entry_tid=42387], [object_number=118, tag_tid=8, entry_tid=42386], [object_number=117, tag_tid=8, entry_tid=42385], [object_number=116, tag_tid=8, entry_tid=42384]]], state=connected] 2026-02-20T21:10:09.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:09.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:09.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:09.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=119, tag_tid=8, entry_tid=42387], [object_number=118, tag_tid=8, entry_tid=42386], [object_number=117, tag_tid=8, entry_tid=42385], [object_number=116, tag_tid=8, entry_tid=42384]]], state=connected] 2026-02-20T21:10:09.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=119, tag_tid=8, entry_tid=42387' 2026-02-20T21:10:09.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=119, tag_tid=8, entry_tid=42387' 2026-02-20T21:10:09.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=119, tag_tid=8, entry_tid=42387' '!=' 'object_number=118, tag_tid=8, entry_tid=42014' 2026-02-20T21:10:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=119, tag_tid=8, entry_tid=42387' = 'object_number=118, tag_tid=8, entry_tid=42014' 2026-02-20T21:10:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=119, tag_tid=8, entry_tid=42387' 2026-02-20T21:10:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=119, tag_tid=8, entry_tid=42387' 2026-02-20T21:10:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:09.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:09.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=119, tag_tid=8, entry_tid=42387' 2026-02-20T21:10:09.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:09.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42387 2026-02-20T21:10:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 42387 2026-02-20T21:10:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:09.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:09.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:09.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:09.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:09.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:09.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:09.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:09.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:09.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:09.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:09.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:09.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:09.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:09.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:09.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:09.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:09.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:09.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:09.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:09.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:09.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:09.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:09.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:09.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:09.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:09.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:09.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:09.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:09.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:09.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:09.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:09.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:09.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:09.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:09.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:09.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:09.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:09.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:09.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:09.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:09.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:09.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:09.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=42758], [object_number=117, tag_tid=8, entry_tid=42757], [object_number=116, tag_tid=8, entry_tid=42756], [object_number=119, tag_tid=8, entry_tid=42755]]], state=connected] 2026-02-20T21:10:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:09.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:09.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:09.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:09.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=42758], [object_number=117, tag_tid=8, entry_tid=42757], [object_number=116, tag_tid=8, entry_tid=42756], [object_number=119, tag_tid=8, entry_tid=42755]]], state=connected] 2026-02-20T21:10:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=118, tag_tid=8, entry_tid=42758' 2026-02-20T21:10:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=118, tag_tid=8, entry_tid=42758' 2026-02-20T21:10:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=42758' '!=' 'object_number=119, tag_tid=8, entry_tid=42387' 2026-02-20T21:10:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=42758' = 'object_number=119, tag_tid=8, entry_tid=42387' 2026-02-20T21:10:09.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=118, tag_tid=8, entry_tid=42758' 2026-02-20T21:10:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:09.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:09.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=42758' 2026-02-20T21:10:09.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:09.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:09.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:09.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:09.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:09.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:09.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=42758' 2026-02-20T21:10:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42758 2026-02-20T21:10:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 42758 2026-02-20T21:10:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:10.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:10.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:10.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:10.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:10.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:10.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:10.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:10.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:10.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:10.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:10.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:10.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:10.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:10.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=119, tag_tid=8, entry_tid=43135], [object_number=118, tag_tid=8, entry_tid=43134], [object_number=117, tag_tid=8, entry_tid=43133], [object_number=116, tag_tid=8, entry_tid=43132]]], state=connected] 2026-02-20T21:10:10.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:10.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=119, tag_tid=8, entry_tid=43135], [object_number=118, tag_tid=8, entry_tid=43134], [object_number=117, tag_tid=8, entry_tid=43133], [object_number=116, tag_tid=8, entry_tid=43132]]], state=connected] 2026-02-20T21:10:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:10.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=119, tag_tid=8, entry_tid=43135' 2026-02-20T21:10:10.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=119, tag_tid=8, entry_tid=43135' 2026-02-20T21:10:10.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=119, tag_tid=8, entry_tid=43135' '!=' 'object_number=118, tag_tid=8, entry_tid=42758' 2026-02-20T21:10:10.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:10.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=119, tag_tid=8, entry_tid=43135' = 'object_number=118, tag_tid=8, entry_tid=42758' 2026-02-20T21:10:10.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=119, tag_tid=8, entry_tid=43135' 2026-02-20T21:10:10.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:10.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:10.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:10.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:10.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=119, tag_tid=8, entry_tid=43135' 2026-02-20T21:10:10.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:10.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=119, tag_tid=8, entry_tid=43135' 2026-02-20T21:10:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:10.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43135 2026-02-20T21:10:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 43135 2026-02-20T21:10:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:10.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:10.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:10.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:10.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:10.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:10.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:10.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:10.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:10.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:10.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:10.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:10.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:10.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:10.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:10.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:10.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:10.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:10.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:10.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:10.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:10.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:10.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:10.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:10.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:10.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:10.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:10.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:10.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:10.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:10.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:10.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:10.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:10.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:10.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:10.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:10.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:10.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:10.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=43508], [object_number=119, tag_tid=8, entry_tid=43507], [object_number=118, tag_tid=8, entry_tid=43506], [object_number=117, tag_tid=8, entry_tid=43505]]], state=connected] 2026-02-20T21:10:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:10.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=43508], [object_number=119, tag_tid=8, entry_tid=43507], [object_number=118, tag_tid=8, entry_tid=43506], [object_number=117, tag_tid=8, entry_tid=43505]]], state=connected] 2026-02-20T21:10:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=116, tag_tid=8, entry_tid=43508' 2026-02-20T21:10:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=116, tag_tid=8, entry_tid=43508' 2026-02-20T21:10:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=43508' '!=' 'object_number=119, tag_tid=8, entry_tid=43135' 2026-02-20T21:10:10.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=43508' = 'object_number=119, tag_tid=8, entry_tid=43135' 2026-02-20T21:10:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=116, tag_tid=8, entry_tid=43508' 2026-02-20T21:10:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:10.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=43508' 2026-02-20T21:10:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:10.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:10.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:10.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=43508' 2026-02-20T21:10:10.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:10.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43508 2026-02-20T21:10:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 43508 2026-02-20T21:10:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:10.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:10.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:10.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:10.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:10.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:10.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:11.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:11.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:11.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:11.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:11.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:11.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:11.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:11.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:11.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:11.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:11.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:11.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:11.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:11.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:11.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:11.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:11.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:11.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:11.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:11.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:11.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:11.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:11.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:11.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:11.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:11.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:11.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:11.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:11.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:11.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=43886], [object_number=117, tag_tid=8, entry_tid=43885], [object_number=116, tag_tid=8, entry_tid=43884], [object_number=119, tag_tid=8, entry_tid=43883]]], state=connected] 2026-02-20T21:10:11.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:11.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:11.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:11.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:11.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:11.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=118, tag_tid=8, entry_tid=43886], [object_number=117, tag_tid=8, entry_tid=43885], [object_number=116, tag_tid=8, entry_tid=43884], [object_number=119, tag_tid=8, entry_tid=43883]]], state=connected] 2026-02-20T21:10:11.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=118, tag_tid=8, entry_tid=43886' 2026-02-20T21:10:11.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=118, tag_tid=8, entry_tid=43886' 2026-02-20T21:10:11.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=43886' '!=' 'object_number=116, tag_tid=8, entry_tid=43508' 2026-02-20T21:10:11.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:11.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=118, tag_tid=8, entry_tid=43886' = 'object_number=116, tag_tid=8, entry_tid=43508' 2026-02-20T21:10:11.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=118, tag_tid=8, entry_tid=43886' 2026-02-20T21:10:11.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:11.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:11.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:11.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:11.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=43886' 2026-02-20T21:10:11.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:11.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:11.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:11.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:11.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:11.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:11.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=118, tag_tid=8, entry_tid=43886' 2026-02-20T21:10:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:11.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43886 2026-02-20T21:10:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 43886 2026-02-20T21:10:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:11.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:11.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:11.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:11.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:11.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:11.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:11.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:11.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:11.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:11.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:11.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:11.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:11.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:11.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:11.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:11.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:11.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:11.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:11.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:11.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:11.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:11.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:11.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:11.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:11.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:11.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:11.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=44257], [object_number=116, tag_tid=8, entry_tid=44256], [object_number=119, tag_tid=8, entry_tid=44255], [object_number=118, tag_tid=8, entry_tid=44254]]], state=connected] 2026-02-20T21:10:11.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:11.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:11.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:11.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:11.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:11.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:11.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:11.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:11.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:11.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:11.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:11.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:11.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:11.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:11.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:11.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=117, tag_tid=8, entry_tid=44257], [object_number=116, tag_tid=8, entry_tid=44256], [object_number=119, tag_tid=8, entry_tid=44255], [object_number=118, tag_tid=8, entry_tid=44254]]], state=connected] 2026-02-20T21:10:11.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=117, tag_tid=8, entry_tid=44257' 2026-02-20T21:10:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=117, tag_tid=8, entry_tid=44257' 2026-02-20T21:10:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=44257' '!=' 'object_number=118, tag_tid=8, entry_tid=43886' 2026-02-20T21:10:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=117, tag_tid=8, entry_tid=44257' = 'object_number=118, tag_tid=8, entry_tid=43886' 2026-02-20T21:10:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=117, tag_tid=8, entry_tid=44257' 2026-02-20T21:10:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=44257' 2026-02-20T21:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:11.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:11.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=117, tag_tid=8, entry_tid=44257' 2026-02-20T21:10:11.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:11.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44257 2026-02-20T21:10:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 44257 2026-02-20T21:10:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:11.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:11.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:11.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:11.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:11.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:11.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:11.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:11.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:11.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:11.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:11.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:11.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:11.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:11.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:11.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:11.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:11.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:11.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:11.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:11.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:11.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:11.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:11.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=44628], [object_number=119, tag_tid=8, entry_tid=44627], [object_number=118, tag_tid=8, entry_tid=44626], [object_number=117, tag_tid=8, entry_tid=44625]]], state=connected] 2026-02-20T21:10:11.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:11.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:11.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:11.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:11.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:11.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:11.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:11.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:11.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 29 2026-02-20T21:10:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=116, tag_tid=8, entry_tid=44628], [object_number=119, tag_tid=8, entry_tid=44627], [object_number=118, tag_tid=8, entry_tid=44626], [object_number=117, tag_tid=8, entry_tid=44625]]], state=connected] 2026-02-20T21:10:11.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=116, tag_tid=8, entry_tid=44628' 2026-02-20T21:10:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=116, tag_tid=8, entry_tid=44628' 2026-02-20T21:10:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=44628' '!=' 'object_number=117, tag_tid=8, entry_tid=44257' 2026-02-20T21:10:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=116, tag_tid=8, entry_tid=44628' = 'object_number=117, tag_tid=8, entry_tid=44257' 2026-02-20T21:10:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=116, tag_tid=8, entry_tid=44628' 2026-02-20T21:10:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:11.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:11.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=44628' 2026-02-20T21:10:11.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:11.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:11.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:11.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:11.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:11.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:11.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=116, tag_tid=8, entry_tid=44628' 2026-02-20T21:10:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:11.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44628 2026-02-20T21:10:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 44628 2026-02-20T21:10:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:12.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:12.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:12.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:12.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:12.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:12.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:12.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:12.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:12.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:12.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:12.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:12.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:12.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:12.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:12.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:12.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:12.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:12.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:12.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:12.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:12.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:12.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:12.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:12.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:12.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:12.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:12.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:12.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:12.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:12.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:12.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:12.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=44900], [object_number=123, tag_tid=8, entry_tid=44899], [object_number=122, tag_tid=8, entry_tid=44898], [object_number=121, tag_tid=8, entry_tid=44897]]], state=connected] 2026-02-20T21:10:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:12.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:12.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:12.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:12.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:12.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:12.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:12.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:12.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:12.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=44900], [object_number=123, tag_tid=8, entry_tid=44899], [object_number=122, tag_tid=8, entry_tid=44898], [object_number=121, tag_tid=8, entry_tid=44897]]], state=connected] 2026-02-20T21:10:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:12.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=120, tag_tid=8, entry_tid=44900' 2026-02-20T21:10:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=120, tag_tid=8, entry_tid=44900' 2026-02-20T21:10:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=44900' '!=' 'object_number=116, tag_tid=8, entry_tid=44628' 2026-02-20T21:10:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=44900' = 'object_number=116, tag_tid=8, entry_tid=44628' 2026-02-20T21:10:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=120, tag_tid=8, entry_tid=44900' 2026-02-20T21:10:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:12.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:12.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=44900' 2026-02-20T21:10:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:12.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:12.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:12.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:12.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=44900' 2026-02-20T21:10:12.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:12.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:12.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44900 2026-02-20T21:10:12.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 44900 2026-02-20T21:10:12.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:12.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:12.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:12.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:12.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:12.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:12.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:12.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:12.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:12.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:12.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:12.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:12.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:12.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:12.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:12.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:12.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:12.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:12.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:12.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:12.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:12.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:12.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:12.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:12.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:12.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:12.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:12.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:12.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:12.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:12.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:12.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:12.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:12.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:12.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:12.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:12.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:12.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:12.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:12.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:12.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:12.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:12.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:12.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:12.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:12.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:12.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:12.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=45279], [object_number=122, tag_tid=8, entry_tid=45278], [object_number=121, tag_tid=8, entry_tid=45277], [object_number=120, tag_tid=8, entry_tid=45276]]], state=connected] 2026-02-20T21:10:12.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:12.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:12.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:12.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:12.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:12.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:12.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:12.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:12.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:12.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:12.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:12.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:12.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:12.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:12.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:12.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:12.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:12.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:12.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:12.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=45279], [object_number=122, tag_tid=8, entry_tid=45278], [object_number=121, tag_tid=8, entry_tid=45277], [object_number=120, tag_tid=8, entry_tid=45276]]], state=connected] 2026-02-20T21:10:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:12.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=123, tag_tid=8, entry_tid=45279' 2026-02-20T21:10:12.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=123, tag_tid=8, entry_tid=45279' 2026-02-20T21:10:12.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=45279' '!=' 'object_number=120, tag_tid=8, entry_tid=44900' 2026-02-20T21:10:12.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:12.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=45279' = 'object_number=120, tag_tid=8, entry_tid=44900' 2026-02-20T21:10:12.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=123, tag_tid=8, entry_tid=45279' 2026-02-20T21:10:12.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:12.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:12.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:12.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:12.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=45279' 2026-02-20T21:10:12.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:12.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:12.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:12.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:12.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:12.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=45279' 2026-02-20T21:10:12.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:12.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45279 2026-02-20T21:10:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 45279 2026-02-20T21:10:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:12.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:12.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:12.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:12.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:12.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:12.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:12.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:12.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:12.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:12.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:12.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:12.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:12.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:12.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:12.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:12.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:13.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:13.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=45656], [object_number=123, tag_tid=8, entry_tid=45655], [object_number=122, tag_tid=8, entry_tid=45654], [object_number=121, tag_tid=8, entry_tid=45653]]], state=connected] 2026-02-20T21:10:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:13.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:13.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:13.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:13.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:13.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:13.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:13.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=45656], [object_number=123, tag_tid=8, entry_tid=45655], [object_number=122, tag_tid=8, entry_tid=45654], [object_number=121, tag_tid=8, entry_tid=45653]]], state=connected] 2026-02-20T21:10:13.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=120, tag_tid=8, entry_tid=45656' 2026-02-20T21:10:13.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=120, tag_tid=8, entry_tid=45656' 2026-02-20T21:10:13.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=45656' '!=' 'object_number=123, tag_tid=8, entry_tid=45279' 2026-02-20T21:10:13.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:13.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=45656' = 'object_number=123, tag_tid=8, entry_tid=45279' 2026-02-20T21:10:13.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=120, tag_tid=8, entry_tid=45656' 2026-02-20T21:10:13.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:13.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:13.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=45656' 2026-02-20T21:10:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:13.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:13.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:13.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=45656' 2026-02-20T21:10:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:13.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45656 2026-02-20T21:10:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 45656 2026-02-20T21:10:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:13.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:13.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:13.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:13.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:13.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:13.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:13.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:13.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:13.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:13.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:13.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:13.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:13.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:13.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:13.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:13.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:13.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:13.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:13.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:13.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:13.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:13.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:13.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:13.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:13.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:13.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:13.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:13.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:13.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:13.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:13.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:13.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:13.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:13.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:13.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:13.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:13.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:13.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:13.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:13.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:13.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:13.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:13.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:13.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:13.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:13.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:13.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:13.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=121, tag_tid=8, entry_tid=46025], [object_number=120, tag_tid=8, entry_tid=46024], [object_number=123, tag_tid=8, entry_tid=46023], [object_number=122, tag_tid=8, entry_tid=46022]]], state=connected] 2026-02-20T21:10:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:13.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:13.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:13.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:13.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:13.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:13.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:13.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:13.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=121, tag_tid=8, entry_tid=46025], [object_number=120, tag_tid=8, entry_tid=46024], [object_number=123, tag_tid=8, entry_tid=46023], [object_number=122, tag_tid=8, entry_tid=46022]]], state=connected] 2026-02-20T21:10:13.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=121, tag_tid=8, entry_tid=46025' 2026-02-20T21:10:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=121, tag_tid=8, entry_tid=46025' 2026-02-20T21:10:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=121, tag_tid=8, entry_tid=46025' '!=' 'object_number=120, tag_tid=8, entry_tid=45656' 2026-02-20T21:10:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=121, tag_tid=8, entry_tid=46025' = 'object_number=120, tag_tid=8, entry_tid=45656' 2026-02-20T21:10:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=121, tag_tid=8, entry_tid=46025' 2026-02-20T21:10:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:13.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=121, tag_tid=8, entry_tid=46025' 2026-02-20T21:10:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:13.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:13.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:13.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:13.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:13.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:13.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=121, tag_tid=8, entry_tid=46025' 2026-02-20T21:10:13.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:13.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:13.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46025 2026-02-20T21:10:13.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 46025 2026-02-20T21:10:13.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:13.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:13.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:13.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:13.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:13.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:13.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:13.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:13.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:13.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:13.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:13.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:13.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:13.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:13.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:13.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:13.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:13.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:13.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:13.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:13.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:13.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:13.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:13.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:13.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:13.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:13.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:13.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:13.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:13.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:13.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:13.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:13.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:13.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:13.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:13.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:13.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:13.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:13.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:13.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:13.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:13.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:13.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:13.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=46388], [object_number=123, tag_tid=8, entry_tid=46387], [object_number=122, tag_tid=8, entry_tid=46386], [object_number=121, tag_tid=8, entry_tid=46385]]], state=connected] 2026-02-20T21:10:13.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:13.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:13.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:13.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:13.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:13.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:13.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:13.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=46388], [object_number=123, tag_tid=8, entry_tid=46387], [object_number=122, tag_tid=8, entry_tid=46386], [object_number=121, tag_tid=8, entry_tid=46385]]], state=connected] 2026-02-20T21:10:13.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=120, tag_tid=8, entry_tid=46388' 2026-02-20T21:10:13.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=120, tag_tid=8, entry_tid=46388' 2026-02-20T21:10:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=46388' '!=' 'object_number=121, tag_tid=8, entry_tid=46025' 2026-02-20T21:10:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=46388' = 'object_number=121, tag_tid=8, entry_tid=46025' 2026-02-20T21:10:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=120, tag_tid=8, entry_tid=46388' 2026-02-20T21:10:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:13.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=46388' 2026-02-20T21:10:13.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:13.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:13.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=46388' 2026-02-20T21:10:13.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:13.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:13.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46388 2026-02-20T21:10:13.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 46388 2026-02-20T21:10:13.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:13.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:13.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:14.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:14.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:14.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:14.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:14.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:14.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:14.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:14.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:14.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:14.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:14.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:14.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:14.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:14.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:14.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=46740], [object_number=123, tag_tid=8, entry_tid=46739], [object_number=122, tag_tid=8, entry_tid=46738], [object_number=121, tag_tid=8, entry_tid=46737]]], state=connected] 2026-02-20T21:10:14.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:14.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:14.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:14.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:14.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=46740], [object_number=123, tag_tid=8, entry_tid=46739], [object_number=122, tag_tid=8, entry_tid=46738], [object_number=121, tag_tid=8, entry_tid=46737]]], state=connected] 2026-02-20T21:10:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=120, tag_tid=8, entry_tid=46740' 2026-02-20T21:10:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=120, tag_tid=8, entry_tid=46740' 2026-02-20T21:10:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=46740' '!=' 'object_number=120, tag_tid=8, entry_tid=46388' 2026-02-20T21:10:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=46740' = 'object_number=120, tag_tid=8, entry_tid=46388' 2026-02-20T21:10:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=120, tag_tid=8, entry_tid=46740' 2026-02-20T21:10:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=46740' 2026-02-20T21:10:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:14.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=46740' 2026-02-20T21:10:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46740 2026-02-20T21:10:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 46740 2026-02-20T21:10:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:14.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:14.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:14.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:14.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:14.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:14.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:14.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:14.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:14.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:14.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:14.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:14.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:14.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:14.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:14.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:14.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:14.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:14.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:14.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:14.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:14.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:14.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:14.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:14.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:14.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:14.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:14.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:14.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:14.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:14.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:14.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:14.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:14.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:14.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:14.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=47099], [object_number=122, tag_tid=8, entry_tid=47098], [object_number=121, tag_tid=8, entry_tid=47097], [object_number=120, tag_tid=8, entry_tid=47096]]], state=connected] 2026-02-20T21:10:14.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:14.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:14.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:14.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:14.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:14.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=47099], [object_number=122, tag_tid=8, entry_tid=47098], [object_number=121, tag_tid=8, entry_tid=47097], [object_number=120, tag_tid=8, entry_tid=47096]]], state=connected] 2026-02-20T21:10:14.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=123, tag_tid=8, entry_tid=47099' 2026-02-20T21:10:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=123, tag_tid=8, entry_tid=47099' 2026-02-20T21:10:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=47099' '!=' 'object_number=120, tag_tid=8, entry_tid=46740' 2026-02-20T21:10:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=47099' = 'object_number=120, tag_tid=8, entry_tid=46740' 2026-02-20T21:10:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=123, tag_tid=8, entry_tid=47099' 2026-02-20T21:10:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=47099' 2026-02-20T21:10:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:14.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:14.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=47099' 2026-02-20T21:10:14.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:14.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47099 2026-02-20T21:10:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 47099 2026-02-20T21:10:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:14.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:14.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:14.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:14.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:14.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:14.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:14.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:14.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:14.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:14.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:14.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:14.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:14.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:14.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:14.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:14.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:14.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:14.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:15.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:15.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:15.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:15.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:15.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:15.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:15.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:15.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:15.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:15.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=47447], [object_number=122, tag_tid=8, entry_tid=47446], [object_number=121, tag_tid=8, entry_tid=47445], [object_number=120, tag_tid=8, entry_tid=47444]]], state=connected] 2026-02-20T21:10:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:15.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:15.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:15.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:15.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:15.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=47447], [object_number=122, tag_tid=8, entry_tid=47446], [object_number=121, tag_tid=8, entry_tid=47445], [object_number=120, tag_tid=8, entry_tid=47444]]], state=connected] 2026-02-20T21:10:15.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=123, tag_tid=8, entry_tid=47447' 2026-02-20T21:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=123, tag_tid=8, entry_tid=47447' 2026-02-20T21:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=47447' '!=' 'object_number=123, tag_tid=8, entry_tid=47099' 2026-02-20T21:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:15.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=47447' = 'object_number=123, tag_tid=8, entry_tid=47099' 2026-02-20T21:10:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=123, tag_tid=8, entry_tid=47447' 2026-02-20T21:10:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:15.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:15.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=47447' 2026-02-20T21:10:15.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:15.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:15.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:15.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:15.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:15.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:15.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:15.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=47447' 2026-02-20T21:10:15.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:15.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:15.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47447 2026-02-20T21:10:15.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 47447 2026-02-20T21:10:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:15.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:15.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:15.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:15.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:15.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:15.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:15.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:15.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:15.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:15.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:15.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:15.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:15.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:15.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=121, tag_tid=8, entry_tid=47805], [object_number=120, tag_tid=8, entry_tid=47804], [object_number=123, tag_tid=8, entry_tid=47803], [object_number=122, tag_tid=8, entry_tid=47802]]], state=connected] 2026-02-20T21:10:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:15.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:15.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:15.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:15.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:15.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:15.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:15.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:15.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:15.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:15.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:15.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:15.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:15.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:15.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:15.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:15.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:15.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=121, tag_tid=8, entry_tid=47805], [object_number=120, tag_tid=8, entry_tid=47804], [object_number=123, tag_tid=8, entry_tid=47803], [object_number=122, tag_tid=8, entry_tid=47802]]], state=connected] 2026-02-20T21:10:15.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=121, tag_tid=8, entry_tid=47805' 2026-02-20T21:10:15.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=121, tag_tid=8, entry_tid=47805' 2026-02-20T21:10:15.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=121, tag_tid=8, entry_tid=47805' '!=' 'object_number=123, tag_tid=8, entry_tid=47447' 2026-02-20T21:10:15.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:15.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=121, tag_tid=8, entry_tid=47805' = 'object_number=123, tag_tid=8, entry_tid=47447' 2026-02-20T21:10:15.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=121, tag_tid=8, entry_tid=47805' 2026-02-20T21:10:15.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:15.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:15.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:15.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:15.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=121, tag_tid=8, entry_tid=47805' 2026-02-20T21:10:15.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:15.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:15.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:15.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:15.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:15.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=121, tag_tid=8, entry_tid=47805' 2026-02-20T21:10:15.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:15.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:15.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47805 2026-02-20T21:10:15.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 47805 2026-02-20T21:10:15.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:15.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:15.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:15.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:15.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:15.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:15.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:15.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:15.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:15.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:15.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:15.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:15.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:15.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:15.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:15.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:15.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:15.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=48140], [object_number=123, tag_tid=8, entry_tid=48139], [object_number=122, tag_tid=8, entry_tid=48138], [object_number=121, tag_tid=8, entry_tid=48137]]], state=connected] 2026-02-20T21:10:15.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:15.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:15.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:15.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:15.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:15.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=120, tag_tid=8, entry_tid=48140], [object_number=123, tag_tid=8, entry_tid=48139], [object_number=122, tag_tid=8, entry_tid=48138], [object_number=121, tag_tid=8, entry_tid=48137]]], state=connected] 2026-02-20T21:10:15.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:15.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=120, tag_tid=8, entry_tid=48140' 2026-02-20T21:10:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=120, tag_tid=8, entry_tid=48140' 2026-02-20T21:10:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=48140' '!=' 'object_number=121, tag_tid=8, entry_tid=47805' 2026-02-20T21:10:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=120, tag_tid=8, entry_tid=48140' = 'object_number=121, tag_tid=8, entry_tid=47805' 2026-02-20T21:10:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=120, tag_tid=8, entry_tid=48140' 2026-02-20T21:10:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:15.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:15.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:15.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=48140' 2026-02-20T21:10:15.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:15.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=120, tag_tid=8, entry_tid=48140' 2026-02-20T21:10:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:15.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48140 2026-02-20T21:10:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 48140 2026-02-20T21:10:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:15.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:16.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:16.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:16.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:16.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:16.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:16.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:16.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:16.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:16.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:16.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:16.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:16.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:16.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:16.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:16.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:16.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:16.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:16.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:16.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:16.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:16.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=48503], [object_number=122, tag_tid=8, entry_tid=48502], [object_number=121, tag_tid=8, entry_tid=48501], [object_number=120, tag_tid=8, entry_tid=48500]]], state=connected] 2026-02-20T21:10:16.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:16.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=123, tag_tid=8, entry_tid=48503], [object_number=122, tag_tid=8, entry_tid=48502], [object_number=121, tag_tid=8, entry_tid=48501], [object_number=120, tag_tid=8, entry_tid=48500]]], state=connected] 2026-02-20T21:10:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=123, tag_tid=8, entry_tid=48503' 2026-02-20T21:10:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=123, tag_tid=8, entry_tid=48503' 2026-02-20T21:10:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=48503' '!=' 'object_number=120, tag_tid=8, entry_tid=48140' 2026-02-20T21:10:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=123, tag_tid=8, entry_tid=48503' = 'object_number=120, tag_tid=8, entry_tid=48140' 2026-02-20T21:10:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=123, tag_tid=8, entry_tid=48503' 2026-02-20T21:10:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=48503' 2026-02-20T21:10:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:16.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=123, tag_tid=8, entry_tid=48503' 2026-02-20T21:10:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:16.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48503 2026-02-20T21:10:16.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 48503 2026-02-20T21:10:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:16.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:16.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:16.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:16.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:16.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:16.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:16.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:16.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:16.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:16.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:16.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:16.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:16.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:16.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:16.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:16.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:16.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:16.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:16.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:16.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:16.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:16.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=126, tag_tid=8, entry_tid=48814], [object_number=125, tag_tid=8, entry_tid=48813], [object_number=124, tag_tid=8, entry_tid=48812], [object_number=123, tag_tid=8, entry_tid=48811]]], state=connected] 2026-02-20T21:10:16.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:16.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:16.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:16.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 30 2026-02-20T21:10:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:16.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:16.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:16.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=126, tag_tid=8, entry_tid=48814], [object_number=125, tag_tid=8, entry_tid=48813], [object_number=124, tag_tid=8, entry_tid=48812], [object_number=123, tag_tid=8, entry_tid=48811]]], state=connected] 2026-02-20T21:10:16.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=126, tag_tid=8, entry_tid=48814' 2026-02-20T21:10:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=126, tag_tid=8, entry_tid=48814' 2026-02-20T21:10:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=126, tag_tid=8, entry_tid=48814' '!=' 'object_number=123, tag_tid=8, entry_tid=48503' 2026-02-20T21:10:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=126, tag_tid=8, entry_tid=48814' = 'object_number=123, tag_tid=8, entry_tid=48503' 2026-02-20T21:10:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=126, tag_tid=8, entry_tid=48814' 2026-02-20T21:10:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=126, tag_tid=8, entry_tid=48814' 2026-02-20T21:10:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:16.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:16.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:16.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:16.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:16.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:16.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=126, tag_tid=8, entry_tid=48814' 2026-02-20T21:10:16.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:16.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48814 2026-02-20T21:10:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 48814 2026-02-20T21:10:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:16.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:16.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:16.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:16.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:16.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:16.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:16.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:16.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:16.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=49215], [object_number=126, tag_tid=8, entry_tid=49214], [object_number=125, tag_tid=8, entry_tid=49213], [object_number=124, tag_tid=8, entry_tid=49212]]], state=connected] 2026-02-20T21:10:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:17.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:17.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:17.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:17.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:17.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:17.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:17.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:17.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=49215], [object_number=126, tag_tid=8, entry_tid=49214], [object_number=125, tag_tid=8, entry_tid=49213], [object_number=124, tag_tid=8, entry_tid=49212]]], state=connected] 2026-02-20T21:10:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=127, tag_tid=8, entry_tid=49215' 2026-02-20T21:10:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=127, tag_tid=8, entry_tid=49215' 2026-02-20T21:10:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=49215' '!=' 'object_number=126, tag_tid=8, entry_tid=48814' 2026-02-20T21:10:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=49215' = 'object_number=126, tag_tid=8, entry_tid=48814' 2026-02-20T21:10:17.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=127, tag_tid=8, entry_tid=49215' 2026-02-20T21:10:17.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:17.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:17.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=49215' 2026-02-20T21:10:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:17.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:17.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:17.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:17.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:17.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=49215' 2026-02-20T21:10:17.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:17.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:17.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49215 2026-02-20T21:10:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 49215 2026-02-20T21:10:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:17.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:17.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:17.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:17.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:17.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:17.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:17.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:17.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:17.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:17.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:17.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:17.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:17.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=49645], [object_number=124, tag_tid=8, entry_tid=49644], [object_number=127, tag_tid=8, entry_tid=49643], [object_number=126, tag_tid=8, entry_tid=49642]]], state=connected] 2026-02-20T21:10:17.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:17.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:17.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:17.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:17.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:17.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=49645], [object_number=124, tag_tid=8, entry_tid=49644], [object_number=127, tag_tid=8, entry_tid=49643], [object_number=126, tag_tid=8, entry_tid=49642]]], state=connected] 2026-02-20T21:10:17.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=125, tag_tid=8, entry_tid=49645' 2026-02-20T21:10:17.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=125, tag_tid=8, entry_tid=49645' 2026-02-20T21:10:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=49645' '!=' 'object_number=127, tag_tid=8, entry_tid=49215' 2026-02-20T21:10:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=49645' = 'object_number=127, tag_tid=8, entry_tid=49215' 2026-02-20T21:10:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=125, tag_tid=8, entry_tid=49645' 2026-02-20T21:10:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:17.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=49645' 2026-02-20T21:10:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:17.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:17.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=49645' 2026-02-20T21:10:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49645 2026-02-20T21:10:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 49645 2026-02-20T21:10:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:17.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:17.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:17.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:17.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:17.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:17.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:17.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:17.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:17.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:17.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:17.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:17.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:17.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:17.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:17.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:17.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=50041], [object_number=124, tag_tid=8, entry_tid=50040], [object_number=127, tag_tid=8, entry_tid=50039], [object_number=126, tag_tid=8, entry_tid=50038]]], state=connected] 2026-02-20T21:10:17.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=50041], [object_number=124, tag_tid=8, entry_tid=50040], [object_number=127, tag_tid=8, entry_tid=50039], [object_number=126, tag_tid=8, entry_tid=50038]]], state=connected] 2026-02-20T21:10:17.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=125, tag_tid=8, entry_tid=50041' 2026-02-20T21:10:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=125, tag_tid=8, entry_tid=50041' 2026-02-20T21:10:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=50041' '!=' 'object_number=125, tag_tid=8, entry_tid=49645' 2026-02-20T21:10:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=50041' = 'object_number=125, tag_tid=8, entry_tid=49645' 2026-02-20T21:10:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=125, tag_tid=8, entry_tid=50041' 2026-02-20T21:10:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=50041' 2026-02-20T21:10:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=50041' 2026-02-20T21:10:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:17.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50041 2026-02-20T21:10:17.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 50041 2026-02-20T21:10:17.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:17.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:17.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:18.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:18.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:18.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:18.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:18.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:18.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:18.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:18.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:18.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:18.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:18.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:18.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:18.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:18.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=126, tag_tid=8, entry_tid=50438], [object_number=125, tag_tid=8, entry_tid=50437], [object_number=124, tag_tid=8, entry_tid=50436], [object_number=127, tag_tid=8, entry_tid=50435]]], state=connected] 2026-02-20T21:10:18.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:18.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:18.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:18.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:18.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:18.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:18.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:18.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:18.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:18.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:18.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:18.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:18.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:18.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:18.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=126, tag_tid=8, entry_tid=50438], [object_number=125, tag_tid=8, entry_tid=50437], [object_number=124, tag_tid=8, entry_tid=50436], [object_number=127, tag_tid=8, entry_tid=50435]]], state=connected] 2026-02-20T21:10:18.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=126, tag_tid=8, entry_tid=50438' 2026-02-20T21:10:18.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=126, tag_tid=8, entry_tid=50438' 2026-02-20T21:10:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=126, tag_tid=8, entry_tid=50438' '!=' 'object_number=125, tag_tid=8, entry_tid=50041' 2026-02-20T21:10:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=126, tag_tid=8, entry_tid=50438' = 'object_number=125, tag_tid=8, entry_tid=50041' 2026-02-20T21:10:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=126, tag_tid=8, entry_tid=50438' 2026-02-20T21:10:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=126, tag_tid=8, entry_tid=50438' 2026-02-20T21:10:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:18.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=126, tag_tid=8, entry_tid=50438' 2026-02-20T21:10:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:18.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50438 2026-02-20T21:10:18.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 50438 2026-02-20T21:10:18.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:18.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:18.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:18.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:18.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:18.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:18.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:18.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:18.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:18.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:18.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:18.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:18.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:18.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:18.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:18.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:18.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=50843], [object_number=126, tag_tid=8, entry_tid=50842], [object_number=125, tag_tid=8, entry_tid=50841], [object_number=124, tag_tid=8, entry_tid=50840]]], state=connected] 2026-02-20T21:10:18.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:18.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:18.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=50843], [object_number=126, tag_tid=8, entry_tid=50842], [object_number=125, tag_tid=8, entry_tid=50841], [object_number=124, tag_tid=8, entry_tid=50840]]], state=connected] 2026-02-20T21:10:18.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=127, tag_tid=8, entry_tid=50843' 2026-02-20T21:10:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=127, tag_tid=8, entry_tid=50843' 2026-02-20T21:10:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=50843' '!=' 'object_number=126, tag_tid=8, entry_tid=50438' 2026-02-20T21:10:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=50843' = 'object_number=126, tag_tid=8, entry_tid=50438' 2026-02-20T21:10:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=127, tag_tid=8, entry_tid=50843' 2026-02-20T21:10:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=50843' 2026-02-20T21:10:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:18.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:18.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=50843' 2026-02-20T21:10:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:18.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:18.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50843 2026-02-20T21:10:18.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 50843 2026-02-20T21:10:18.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:18.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:18.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:18.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:18.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:18.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:18.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:18.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:18.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:18.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:18.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:18.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:18.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:18.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:18.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:18.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:18.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:18.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=51229], [object_number=124, tag_tid=8, entry_tid=51228], [object_number=127, tag_tid=8, entry_tid=51227], [object_number=126, tag_tid=8, entry_tid=51226]]], state=connected] 2026-02-20T21:10:18.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:19.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:19.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:19.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:19.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=51229], [object_number=124, tag_tid=8, entry_tid=51228], [object_number=127, tag_tid=8, entry_tid=51227], [object_number=126, tag_tid=8, entry_tid=51226]]], state=connected] 2026-02-20T21:10:19.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=125, tag_tid=8, entry_tid=51229' 2026-02-20T21:10:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=125, tag_tid=8, entry_tid=51229' 2026-02-20T21:10:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=51229' '!=' 'object_number=127, tag_tid=8, entry_tid=50843' 2026-02-20T21:10:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=51229' = 'object_number=127, tag_tid=8, entry_tid=50843' 2026-02-20T21:10:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=125, tag_tid=8, entry_tid=51229' 2026-02-20T21:10:19.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:19.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:19.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:19.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:19.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=51229' 2026-02-20T21:10:19.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:19.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:19.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=51229' 2026-02-20T21:10:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:19.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51229 2026-02-20T21:10:19.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 51229 2026-02-20T21:10:19.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:19.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:19.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:19.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:19.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:19.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:19.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:19.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:19.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:19.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:19.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:19.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:19.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:19.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:19.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:19.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:19.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:19.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:19.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:19.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:19.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:19.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:19.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:19.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:19.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:19.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:19.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:19.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=51619], [object_number=126, tag_tid=8, entry_tid=51618], [object_number=125, tag_tid=8, entry_tid=51617], [object_number=124, tag_tid=8, entry_tid=51616]]], state=connected] 2026-02-20T21:10:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:19.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:19.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:19.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:19.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:19.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:19.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:19.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:19.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:19.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:19.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:19.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:19.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=51619], [object_number=126, tag_tid=8, entry_tid=51618], [object_number=125, tag_tid=8, entry_tid=51617], [object_number=124, tag_tid=8, entry_tid=51616]]], state=connected] 2026-02-20T21:10:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=127, tag_tid=8, entry_tid=51619' 2026-02-20T21:10:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=127, tag_tid=8, entry_tid=51619' 2026-02-20T21:10:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=51619' '!=' 'object_number=125, tag_tid=8, entry_tid=51229' 2026-02-20T21:10:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:19.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=51619' = 'object_number=125, tag_tid=8, entry_tid=51229' 2026-02-20T21:10:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=127, tag_tid=8, entry_tid=51619' 2026-02-20T21:10:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=51619' 2026-02-20T21:10:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:19.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:19.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:19.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:19.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=51619' 2026-02-20T21:10:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:19.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51619 2026-02-20T21:10:19.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 51619 2026-02-20T21:10:19.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:19.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:19.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:19.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:19.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:19.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:19.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:19.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:19.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:19.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:19.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:19.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:19.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:19.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:19.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:19.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:19.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:19.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:19.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:19.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:19.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:19.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:19.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:19.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:19.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:19.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:19.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:19.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:19.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:19.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:19.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:19.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:19.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:19.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:19.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:19.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:19.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=124, tag_tid=8, entry_tid=52028], [object_number=127, tag_tid=8, entry_tid=52027], [object_number=126, tag_tid=8, entry_tid=52026], [object_number=125, tag_tid=8, entry_tid=52025]]], state=connected] 2026-02-20T21:10:19.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:19.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:19.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:19.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:19.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=124, tag_tid=8, entry_tid=52028], [object_number=127, tag_tid=8, entry_tid=52027], [object_number=126, tag_tid=8, entry_tid=52026], [object_number=125, tag_tid=8, entry_tid=52025]]], state=connected] 2026-02-20T21:10:19.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=124, tag_tid=8, entry_tid=52028' 2026-02-20T21:10:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=124, tag_tid=8, entry_tid=52028' 2026-02-20T21:10:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=124, tag_tid=8, entry_tid=52028' '!=' 'object_number=127, tag_tid=8, entry_tid=51619' 2026-02-20T21:10:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=124, tag_tid=8, entry_tid=52028' = 'object_number=127, tag_tid=8, entry_tid=51619' 2026-02-20T21:10:19.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=124, tag_tid=8, entry_tid=52028' 2026-02-20T21:10:19.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:19.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:19.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:19.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:19.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=124, tag_tid=8, entry_tid=52028' 2026-02-20T21:10:19.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:19.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:19.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:19.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=124, tag_tid=8, entry_tid=52028' 2026-02-20T21:10:19.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:19.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:19.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52028 2026-02-20T21:10:19.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 52028 2026-02-20T21:10:19.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:19.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:19.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:20.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:20.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:20.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:20.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:20.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:20.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:20.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:20.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:20.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:20.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:20.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:20.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:20.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:20.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:20.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=52413], [object_number=124, tag_tid=8, entry_tid=52412], [object_number=127, tag_tid=8, entry_tid=52411], [object_number=126, tag_tid=8, entry_tid=52410]]], state=connected] 2026-02-20T21:10:20.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:20.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:20.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:20.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=125, tag_tid=8, entry_tid=52413], [object_number=124, tag_tid=8, entry_tid=52412], [object_number=127, tag_tid=8, entry_tid=52411], [object_number=126, tag_tid=8, entry_tid=52410]]], state=connected] 2026-02-20T21:10:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=125, tag_tid=8, entry_tid=52413' 2026-02-20T21:10:20.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=125, tag_tid=8, entry_tid=52413' 2026-02-20T21:10:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=52413' '!=' 'object_number=124, tag_tid=8, entry_tid=52028' 2026-02-20T21:10:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=125, tag_tid=8, entry_tid=52413' = 'object_number=124, tag_tid=8, entry_tid=52028' 2026-02-20T21:10:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=125, tag_tid=8, entry_tid=52413' 2026-02-20T21:10:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:20.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=52413' 2026-02-20T21:10:20.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:20.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:20.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:20.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:20.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:20.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:20.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:20.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=125, tag_tid=8, entry_tid=52413' 2026-02-20T21:10:20.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:20.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52413 2026-02-20T21:10:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 52413 2026-02-20T21:10:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:20.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:20.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:20.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:20.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:20.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:20.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:20.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:20.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:20.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:20.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:20.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:20.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:20.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:20.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:20.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:20.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:20.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:20.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:20.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:20.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:20.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:20.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:20.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:20.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:20.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:20.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:20.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:20.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:20.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:20.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:20.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:20.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:20.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=52783], [object_number=126, tag_tid=8, entry_tid=52782], [object_number=125, tag_tid=8, entry_tid=52781], [object_number=124, tag_tid=8, entry_tid=52780]]], state=connected] 2026-02-20T21:10:20.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:20.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:20.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:20.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:20.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 31 2026-02-20T21:10:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=127, tag_tid=8, entry_tid=52783], [object_number=126, tag_tid=8, entry_tid=52782], [object_number=125, tag_tid=8, entry_tid=52781], [object_number=124, tag_tid=8, entry_tid=52780]]], state=connected] 2026-02-20T21:10:20.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=127, tag_tid=8, entry_tid=52783' 2026-02-20T21:10:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=127, tag_tid=8, entry_tid=52783' 2026-02-20T21:10:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=52783' '!=' 'object_number=125, tag_tid=8, entry_tid=52413' 2026-02-20T21:10:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=127, tag_tid=8, entry_tid=52783' = 'object_number=125, tag_tid=8, entry_tid=52413' 2026-02-20T21:10:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=127, tag_tid=8, entry_tid=52783' 2026-02-20T21:10:20.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:20.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=52783' 2026-02-20T21:10:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:20.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:20.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:20.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=127, tag_tid=8, entry_tid=52783' 2026-02-20T21:10:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52783 2026-02-20T21:10:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 52783 2026-02-20T21:10:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:20.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:20.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:20.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:20.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:20.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:20.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:20.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:20.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:20.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:20.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:20.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:20.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:20.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:20.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:20.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:20.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:20.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:20.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:20.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:20.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:20.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:20.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:20.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:20.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:20.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:20.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:20.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:20.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:20.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:20.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:20.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:20.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:20.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:20.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:20.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:20.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:20.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:20.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:20.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:20.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:20.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:20.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:20.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:20.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:20.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:20.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:20.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:20.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=53100], [object_number=131, tag_tid=8, entry_tid=53099], [object_number=130, tag_tid=8, entry_tid=53098], [object_number=129, tag_tid=8, entry_tid=53097]]], state=connected] 2026-02-20T21:10:20.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:20.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:20.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:20.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:20.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:20.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:20.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:20.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:20.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:20.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:20.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=53100], [object_number=131, tag_tid=8, entry_tid=53099], [object_number=130, tag_tid=8, entry_tid=53098], [object_number=129, tag_tid=8, entry_tid=53097]]], state=connected] 2026-02-20T21:10:20.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:20.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=128, tag_tid=8, entry_tid=53100' 2026-02-20T21:10:20.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=128, tag_tid=8, entry_tid=53100' 2026-02-20T21:10:20.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=53100' '!=' 'object_number=127, tag_tid=8, entry_tid=52783' 2026-02-20T21:10:20.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:20.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=53100' = 'object_number=127, tag_tid=8, entry_tid=52783' 2026-02-20T21:10:20.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=128, tag_tid=8, entry_tid=53100' 2026-02-20T21:10:20.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:20.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:20.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:20.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:20.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=53100' 2026-02-20T21:10:20.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:20.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:20.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:20.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:20.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:20.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:20.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:20.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=53100' 2026-02-20T21:10:20.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:20.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53100 2026-02-20T21:10:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 53100 2026-02-20T21:10:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:21.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:21.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:21.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:21.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:21.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:21.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:21.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:21.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:21.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:21.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:21.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:21.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:21.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:21.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:21.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:21.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:21.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:21.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:21.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:21.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:21.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:21.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:21.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:21.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:21.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:21.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:21.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:21.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:21.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:21.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:21.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=53443], [object_number=130, tag_tid=8, entry_tid=53442], [object_number=129, tag_tid=8, entry_tid=53441], [object_number=128, tag_tid=8, entry_tid=53440]]], state=connected] 2026-02-20T21:10:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:21.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:21.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:21.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:21.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:21.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:21.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:21.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=53443], [object_number=130, tag_tid=8, entry_tid=53442], [object_number=129, tag_tid=8, entry_tid=53441], [object_number=128, tag_tid=8, entry_tid=53440]]], state=connected] 2026-02-20T21:10:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=53443' 2026-02-20T21:10:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=53443' 2026-02-20T21:10:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=53443' '!=' 'object_number=128, tag_tid=8, entry_tid=53100' 2026-02-20T21:10:21.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=53443' = 'object_number=128, tag_tid=8, entry_tid=53100' 2026-02-20T21:10:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=53443' 2026-02-20T21:10:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=53443' 2026-02-20T21:10:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:21.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:21.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:21.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:21.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:21.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=53443' 2026-02-20T21:10:21.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:21.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:21.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53443 2026-02-20T21:10:21.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 53443 2026-02-20T21:10:21.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:21.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:21.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:21.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:21.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:21.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:21.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:21.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:21.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:21.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:21.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:21.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:21.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:21.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:21.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:21.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:21.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:21.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:21.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:21.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:21.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=53781], [object_number=128, tag_tid=8, entry_tid=53780], [object_number=131, tag_tid=8, entry_tid=53779], [object_number=130, tag_tid=8, entry_tid=53778]]], state=connected] 2026-02-20T21:10:21.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:21.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:21.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:21.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:21.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:21.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:21.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:21.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:21.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:21.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=53781], [object_number=128, tag_tid=8, entry_tid=53780], [object_number=131, tag_tid=8, entry_tid=53779], [object_number=130, tag_tid=8, entry_tid=53778]]], state=connected] 2026-02-20T21:10:21.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=129, tag_tid=8, entry_tid=53781' 2026-02-20T21:10:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=129, tag_tid=8, entry_tid=53781' 2026-02-20T21:10:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=53781' '!=' 'object_number=131, tag_tid=8, entry_tid=53443' 2026-02-20T21:10:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=53781' = 'object_number=131, tag_tid=8, entry_tid=53443' 2026-02-20T21:10:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=129, tag_tid=8, entry_tid=53781' 2026-02-20T21:10:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:21.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=53781' 2026-02-20T21:10:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:21.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:21.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:21.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:21.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=53781' 2026-02-20T21:10:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:21.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:21.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53781 2026-02-20T21:10:21.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 53781 2026-02-20T21:10:21.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:21.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:21.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:21.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:21.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:21.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:21.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:21.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:22.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:22.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:22.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:22.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:22.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:22.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:22.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:22.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:22.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:22.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:22.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:22.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:22.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:22.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:22.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:22.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:22.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:22.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=54151], [object_number=130, tag_tid=8, entry_tid=54150], [object_number=129, tag_tid=8, entry_tid=54149], [object_number=128, tag_tid=8, entry_tid=54148]]], state=connected] 2026-02-20T21:10:22.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:22.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:22.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:22.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:22.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:22.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=54151], [object_number=130, tag_tid=8, entry_tid=54150], [object_number=129, tag_tid=8, entry_tid=54149], [object_number=128, tag_tid=8, entry_tid=54148]]], state=connected] 2026-02-20T21:10:22.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=54151' 2026-02-20T21:10:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=54151' 2026-02-20T21:10:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=54151' '!=' 'object_number=129, tag_tid=8, entry_tid=53781' 2026-02-20T21:10:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=54151' = 'object_number=129, tag_tid=8, entry_tid=53781' 2026-02-20T21:10:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=54151' 2026-02-20T21:10:22.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:22.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=54151' 2026-02-20T21:10:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:22.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:22.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:22.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:22.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:22.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:22.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=54151' 2026-02-20T21:10:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:22.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54151 2026-02-20T21:10:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 54151 2026-02-20T21:10:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:22.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:22.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:22.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:22.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:22.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:22.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:22.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:22.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:22.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:22.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:22.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:22.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:22.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:22.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:22.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:22.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:22.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:22.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:22.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:22.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:22.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:22.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:22.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:22.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:22.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:22.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:22.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:22.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:22.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:22.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:22.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:22.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=54525], [object_number=128, tag_tid=8, entry_tid=54524], [object_number=131, tag_tid=8, entry_tid=54523], [object_number=130, tag_tid=8, entry_tid=54522]]], state=connected] 2026-02-20T21:10:22.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:22.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=54525], [object_number=128, tag_tid=8, entry_tid=54524], [object_number=131, tag_tid=8, entry_tid=54523], [object_number=130, tag_tid=8, entry_tid=54522]]], state=connected] 2026-02-20T21:10:22.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=129, tag_tid=8, entry_tid=54525' 2026-02-20T21:10:22.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=129, tag_tid=8, entry_tid=54525' 2026-02-20T21:10:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=54525' '!=' 'object_number=131, tag_tid=8, entry_tid=54151' 2026-02-20T21:10:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=54525' = 'object_number=131, tag_tid=8, entry_tid=54151' 2026-02-20T21:10:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=129, tag_tid=8, entry_tid=54525' 2026-02-20T21:10:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:22.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=54525' 2026-02-20T21:10:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:22.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:22.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:22.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=54525' 2026-02-20T21:10:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:22.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54525 2026-02-20T21:10:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 54525 2026-02-20T21:10:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:22.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:22.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:22.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:22.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:22.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:22.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:22.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:22.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:22.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:22.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:22.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:22.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:22.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:22.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:22.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:22.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:22.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:22.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:22.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:22.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=54896], [object_number=131, tag_tid=8, entry_tid=54895], [object_number=130, tag_tid=8, entry_tid=54894], [object_number=129, tag_tid=8, entry_tid=54893]]], state=connected] 2026-02-20T21:10:22.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:22.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:22.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=54896], [object_number=131, tag_tid=8, entry_tid=54895], [object_number=130, tag_tid=8, entry_tid=54894], [object_number=129, tag_tid=8, entry_tid=54893]]], state=connected] 2026-02-20T21:10:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=128, tag_tid=8, entry_tid=54896' 2026-02-20T21:10:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=128, tag_tid=8, entry_tid=54896' 2026-02-20T21:10:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=54896' '!=' 'object_number=129, tag_tid=8, entry_tid=54525' 2026-02-20T21:10:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=54896' = 'object_number=129, tag_tid=8, entry_tid=54525' 2026-02-20T21:10:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=128, tag_tid=8, entry_tid=54896' 2026-02-20T21:10:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=54896' 2026-02-20T21:10:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=54896' 2026-02-20T21:10:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:22.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:22.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54896 2026-02-20T21:10:22.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 54896 2026-02-20T21:10:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:22.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:23.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:23.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:23.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:23.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:23.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:23.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:23.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:23.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:23.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:23.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:23.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:23.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:23.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:23.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:23.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:23.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:23.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:23.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:23.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:23.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:23.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:23.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:23.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:23.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:23.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:23.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:23.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:23.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:23.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:23.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:23.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:23.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:23.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=55258], [object_number=129, tag_tid=8, entry_tid=55257], [object_number=128, tag_tid=8, entry_tid=55256], [object_number=131, tag_tid=8, entry_tid=55255]]], state=connected] 2026-02-20T21:10:23.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:23.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=55258], [object_number=129, tag_tid=8, entry_tid=55257], [object_number=128, tag_tid=8, entry_tid=55256], [object_number=131, tag_tid=8, entry_tid=55255]]], state=connected] 2026-02-20T21:10:23.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=130, tag_tid=8, entry_tid=55258' 2026-02-20T21:10:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=130, tag_tid=8, entry_tid=55258' 2026-02-20T21:10:23.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=55258' '!=' 'object_number=128, tag_tid=8, entry_tid=54896' 2026-02-20T21:10:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=55258' = 'object_number=128, tag_tid=8, entry_tid=54896' 2026-02-20T21:10:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=130, tag_tid=8, entry_tid=55258' 2026-02-20T21:10:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=55258' 2026-02-20T21:10:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:23.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:23.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:23.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:23.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:23.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:23.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=55258' 2026-02-20T21:10:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55258 2026-02-20T21:10:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 55258 2026-02-20T21:10:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:23.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:23.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:23.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:23.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:23.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:23.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:23.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:23.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:23.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:23.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:23.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:23.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:23.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:23.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:23.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:23.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:23.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=55675], [object_number=130, tag_tid=8, entry_tid=55674], [object_number=129, tag_tid=8, entry_tid=55673], [object_number=128, tag_tid=8, entry_tid=55672]]], state=connected] 2026-02-20T21:10:23.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:23.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:23.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:23.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:23.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:23.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:23.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:23.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:23.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:23.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=131, tag_tid=8, entry_tid=55675], [object_number=130, tag_tid=8, entry_tid=55674], [object_number=129, tag_tid=8, entry_tid=55673], [object_number=128, tag_tid=8, entry_tid=55672]]], state=connected] 2026-02-20T21:10:23.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=131, tag_tid=8, entry_tid=55675' 2026-02-20T21:10:23.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=131, tag_tid=8, entry_tid=55675' 2026-02-20T21:10:23.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=55675' '!=' 'object_number=130, tag_tid=8, entry_tid=55258' 2026-02-20T21:10:23.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:23.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=131, tag_tid=8, entry_tid=55675' = 'object_number=130, tag_tid=8, entry_tid=55258' 2026-02-20T21:10:23.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=131, tag_tid=8, entry_tid=55675' 2026-02-20T21:10:23.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:23.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:23.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:23.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:23.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=55675' 2026-02-20T21:10:23.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:23.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:23.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:23.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:23.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:23.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:23.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:23.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=131, tag_tid=8, entry_tid=55675' 2026-02-20T21:10:23.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:23.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:23.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55675 2026-02-20T21:10:23.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 55675 2026-02-20T21:10:23.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:23.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:23.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:23.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:23.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:23.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:23.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:23.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:23.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:23.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:23.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:23.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:24.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=56058], [object_number=129, tag_tid=8, entry_tid=56057], [object_number=128, tag_tid=8, entry_tid=56056], [object_number=131, tag_tid=8, entry_tid=56055]]], state=connected] 2026-02-20T21:10:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:24.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=130, tag_tid=8, entry_tid=56058], [object_number=129, tag_tid=8, entry_tid=56057], [object_number=128, tag_tid=8, entry_tid=56056], [object_number=131, tag_tid=8, entry_tid=56055]]], state=connected] 2026-02-20T21:10:24.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=130, tag_tid=8, entry_tid=56058' 2026-02-20T21:10:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=130, tag_tid=8, entry_tid=56058' 2026-02-20T21:10:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=56058' '!=' 'object_number=131, tag_tid=8, entry_tid=55675' 2026-02-20T21:10:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=130, tag_tid=8, entry_tid=56058' = 'object_number=131, tag_tid=8, entry_tid=55675' 2026-02-20T21:10:24.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=130, tag_tid=8, entry_tid=56058' 2026-02-20T21:10:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:24.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=56058' 2026-02-20T21:10:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:24.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:24.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:24.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=130, tag_tid=8, entry_tid=56058' 2026-02-20T21:10:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56058 2026-02-20T21:10:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 56058 2026-02-20T21:10:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:24.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:24.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:24.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:24.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:24.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:24.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:24.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:24.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:24.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:24.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:24.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:24.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:24.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:24.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:24.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:24.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:24.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:24.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:24.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:24.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:24.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:24.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:24.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:24.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:24.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:24.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:24.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:24.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:24.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:24.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:24.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:24.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:24.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:24.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:24.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:24.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:24.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:24.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:24.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:24.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:24.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:24.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:24.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:24.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:24.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:24.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:24.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:24.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:24.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:24.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:24.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:24.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:24.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:24.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:24.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:24.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:24.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:24.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:24.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:24.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:24.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:24.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:24.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:24.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:24.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:24.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:24.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=56436], [object_number=131, tag_tid=8, entry_tid=56435], [object_number=130, tag_tid=8, entry_tid=56434], [object_number=129, tag_tid=8, entry_tid=56433]]], state=connected] 2026-02-20T21:10:24.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:24.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:24.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:24.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:24.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:24.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:24.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:24.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=128, tag_tid=8, entry_tid=56436], [object_number=131, tag_tid=8, entry_tid=56435], [object_number=130, tag_tid=8, entry_tid=56434], [object_number=129, tag_tid=8, entry_tid=56433]]], state=connected] 2026-02-20T21:10:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=128, tag_tid=8, entry_tid=56436' 2026-02-20T21:10:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=128, tag_tid=8, entry_tid=56436' 2026-02-20T21:10:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=56436' '!=' 'object_number=130, tag_tid=8, entry_tid=56058' 2026-02-20T21:10:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=128, tag_tid=8, entry_tid=56436' = 'object_number=130, tag_tid=8, entry_tid=56058' 2026-02-20T21:10:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=128, tag_tid=8, entry_tid=56436' 2026-02-20T21:10:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:24.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=56436' 2026-02-20T21:10:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:24.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:24.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:24.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:24.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=128, tag_tid=8, entry_tid=56436' 2026-02-20T21:10:24.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:24.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:24.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56436 2026-02-20T21:10:24.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 56436 2026-02-20T21:10:24.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:24.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:24.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:24.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:24.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:24.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:24.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:24.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:24.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:24.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:24.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:24.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:24.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:24.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:24.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:24.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:24.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:24.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:24.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:24.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:24.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:24.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:24.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:24.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:24.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=56789], [object_number=128, tag_tid=8, entry_tid=56788], [object_number=131, tag_tid=8, entry_tid=56787], [object_number=130, tag_tid=8, entry_tid=56786]]], state=connected] 2026-02-20T21:10:24.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:24.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:24.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:24.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:24.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:24.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:24.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:24.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:24.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:24.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:24.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:24.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:24.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:24.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:24.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=129, tag_tid=8, entry_tid=56789], [object_number=128, tag_tid=8, entry_tid=56788], [object_number=131, tag_tid=8, entry_tid=56787], [object_number=130, tag_tid=8, entry_tid=56786]]], state=connected] 2026-02-20T21:10:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=129, tag_tid=8, entry_tid=56789' 2026-02-20T21:10:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=129, tag_tid=8, entry_tid=56789' 2026-02-20T21:10:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=56789' '!=' 'object_number=128, tag_tid=8, entry_tid=56436' 2026-02-20T21:10:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=129, tag_tid=8, entry_tid=56789' = 'object_number=128, tag_tid=8, entry_tid=56436' 2026-02-20T21:10:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=129, tag_tid=8, entry_tid=56789' 2026-02-20T21:10:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=56789' 2026-02-20T21:10:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:24.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:24.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=129, tag_tid=8, entry_tid=56789' 2026-02-20T21:10:24.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:24.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56789 2026-02-20T21:10:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 56789 2026-02-20T21:10:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:25.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:25.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:25.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:25.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:25.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:25.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=57094], [object_number=133, tag_tid=8, entry_tid=57093], [object_number=132, tag_tid=8, entry_tid=57092], [object_number=131, tag_tid=8, entry_tid=57091]]], state=connected] 2026-02-20T21:10:25.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:25.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:25.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:25.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:25.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:25.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:25.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 32 2026-02-20T21:10:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=57094], [object_number=133, tag_tid=8, entry_tid=57093], [object_number=132, tag_tid=8, entry_tid=57092], [object_number=131, tag_tid=8, entry_tid=57091]]], state=connected] 2026-02-20T21:10:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=134, tag_tid=8, entry_tid=57094' 2026-02-20T21:10:25.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=134, tag_tid=8, entry_tid=57094' 2026-02-20T21:10:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=57094' '!=' 'object_number=129, tag_tid=8, entry_tid=56789' 2026-02-20T21:10:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=57094' = 'object_number=129, tag_tid=8, entry_tid=56789' 2026-02-20T21:10:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=134, tag_tid=8, entry_tid=57094' 2026-02-20T21:10:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:25.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:25.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:25.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=57094' 2026-02-20T21:10:25.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:25.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:25.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:25.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=57094' 2026-02-20T21:10:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:25.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:25.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57094 2026-02-20T21:10:25.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 57094 2026-02-20T21:10:25.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:25.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:25.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:25.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:25.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:25.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:25.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:25.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:25.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:25.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:25.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:25.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:25.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:25.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:25.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:25.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:25.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:25.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:25.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:25.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:25.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:25.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:25.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:25.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:25.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:25.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:25.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:25.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:25.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:25.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:25.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:25.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:25.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:25.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:25.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:25.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=57441], [object_number=132, tag_tid=8, entry_tid=57440], [object_number=135, tag_tid=8, entry_tid=57439], [object_number=134, tag_tid=8, entry_tid=57438]]], state=connected] 2026-02-20T21:10:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:25.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:25.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:25.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:25.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:25.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=133, tag_tid=8, entry_tid=57441], [object_number=132, tag_tid=8, entry_tid=57440], [object_number=135, tag_tid=8, entry_tid=57439], [object_number=134, tag_tid=8, entry_tid=57438]]], state=connected] 2026-02-20T21:10:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:25.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=133, tag_tid=8, entry_tid=57441' 2026-02-20T21:10:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=133, tag_tid=8, entry_tid=57441' 2026-02-20T21:10:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=57441' '!=' 'object_number=134, tag_tid=8, entry_tid=57094' 2026-02-20T21:10:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=133, tag_tid=8, entry_tid=57441' = 'object_number=134, tag_tid=8, entry_tid=57094' 2026-02-20T21:10:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=133, tag_tid=8, entry_tid=57441' 2026-02-20T21:10:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=57441' 2026-02-20T21:10:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:25.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:25.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:25.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:25.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=8, entry_tid=57441' 2026-02-20T21:10:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:25.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:25.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57441 2026-02-20T21:10:25.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 57441 2026-02-20T21:10:25.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:25.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:25.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:25.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:25.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:25.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:25.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:25.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:25.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:25.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:25.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:25.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:25.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:25.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:25.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:25.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:25.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:25.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:25.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:25.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:25.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:25.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:25.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:25.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:25.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:25.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:25.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:25.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:25.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:25.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:25.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:25.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:25.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:25.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:25.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:25.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:25.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:25.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:25.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:25.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:26.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:26.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:26.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:26.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:26.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:26.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:26.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:26.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:26.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:26.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:26.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=57806], [object_number=133, tag_tid=8, entry_tid=57805], [object_number=132, tag_tid=8, entry_tid=57804], [object_number=135, tag_tid=8, entry_tid=57803]]], state=connected] 2026-02-20T21:10:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:26.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:26.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:26.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:26.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:26.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=57806], [object_number=133, tag_tid=8, entry_tid=57805], [object_number=132, tag_tid=8, entry_tid=57804], [object_number=135, tag_tid=8, entry_tid=57803]]], state=connected] 2026-02-20T21:10:26.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=134, tag_tid=8, entry_tid=57806' 2026-02-20T21:10:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=134, tag_tid=8, entry_tid=57806' 2026-02-20T21:10:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=57806' '!=' 'object_number=133, tag_tid=8, entry_tid=57441' 2026-02-20T21:10:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=57806' = 'object_number=133, tag_tid=8, entry_tid=57441' 2026-02-20T21:10:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=134, tag_tid=8, entry_tid=57806' 2026-02-20T21:10:26.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:26.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:26.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:26.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:26.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=57806' 2026-02-20T21:10:26.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:26.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:26.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:26.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:26.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:26.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:26.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:26.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=57806' 2026-02-20T21:10:26.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:26.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57806 2026-02-20T21:10:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 57806 2026-02-20T21:10:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:26.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:26.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:26.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:26.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:26.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:26.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:26.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:26.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:26.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:26.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:26.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:26.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:26.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:26.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:26.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:26.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:26.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:26.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:26.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=58359], [object_number=134, tag_tid=8, entry_tid=58358], [object_number=133, tag_tid=8, entry_tid=58357], [object_number=132, tag_tid=8, entry_tid=58356]]], state=connected] 2026-02-20T21:10:26.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:26.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:26.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:26.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:26.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:26.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:26.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=58359], [object_number=134, tag_tid=8, entry_tid=58358], [object_number=133, tag_tid=8, entry_tid=58357], [object_number=132, tag_tid=8, entry_tid=58356]]], state=connected] 2026-02-20T21:10:26.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:26.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=135, tag_tid=8, entry_tid=58359' 2026-02-20T21:10:26.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=135, tag_tid=8, entry_tid=58359' 2026-02-20T21:10:26.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=58359' '!=' 'object_number=134, tag_tid=8, entry_tid=57806' 2026-02-20T21:10:26.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:26.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=58359' = 'object_number=134, tag_tid=8, entry_tid=57806' 2026-02-20T21:10:26.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=135, tag_tid=8, entry_tid=58359' 2026-02-20T21:10:26.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:26.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:26.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:26.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:26.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=58359' 2026-02-20T21:10:26.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:26.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:26.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:26.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:26.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:26.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:26.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:26.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=58359' 2026-02-20T21:10:26.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:26.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:26.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58359 2026-02-20T21:10:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 58359 2026-02-20T21:10:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:26.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:26.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:26.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:26.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:26.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:26.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:26.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:26.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:26.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:26.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:26.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:26.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:26.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:26.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:26.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:26.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:26.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:26.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:26.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:26.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:26.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:26.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:27.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:27.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:27.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:27.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=58738], [object_number=133, tag_tid=8, entry_tid=58737], [object_number=132, tag_tid=8, entry_tid=58736], [object_number=135, tag_tid=8, entry_tid=58735]]], state=connected] 2026-02-20T21:10:27.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:27.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:27.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:27.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:27.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:27.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:27.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:27.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:27.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:27.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:27.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:27.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=58738], [object_number=133, tag_tid=8, entry_tid=58737], [object_number=132, tag_tid=8, entry_tid=58736], [object_number=135, tag_tid=8, entry_tid=58735]]], state=connected] 2026-02-20T21:10:27.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=134, tag_tid=8, entry_tid=58738' 2026-02-20T21:10:27.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=134, tag_tid=8, entry_tid=58738' 2026-02-20T21:10:27.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=58738' '!=' 'object_number=135, tag_tid=8, entry_tid=58359' 2026-02-20T21:10:27.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:27.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=58738' = 'object_number=135, tag_tid=8, entry_tid=58359' 2026-02-20T21:10:27.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=134, tag_tid=8, entry_tid=58738' 2026-02-20T21:10:27.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:27.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=58738' 2026-02-20T21:10:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:27.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:27.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:27.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:27.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:27.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:27.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=58738' 2026-02-20T21:10:27.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:27.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:27.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58738 2026-02-20T21:10:27.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 58738 2026-02-20T21:10:27.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:27.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:27.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:27.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:27.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:27.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:27.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:27.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=59103], [object_number=134, tag_tid=8, entry_tid=59102], [object_number=133, tag_tid=8, entry_tid=59101], [object_number=132, tag_tid=8, entry_tid=59100]]], state=connected] 2026-02-20T21:10:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:27.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:27.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:27.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:27.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:27.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:27.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:27.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:27.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:27.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:27.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=59103], [object_number=134, tag_tid=8, entry_tid=59102], [object_number=133, tag_tid=8, entry_tid=59101], [object_number=132, tag_tid=8, entry_tid=59100]]], state=connected] 2026-02-20T21:10:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=135, tag_tid=8, entry_tid=59103' 2026-02-20T21:10:27.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=135, tag_tid=8, entry_tid=59103' 2026-02-20T21:10:27.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=59103' '!=' 'object_number=134, tag_tid=8, entry_tid=58738' 2026-02-20T21:10:27.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:27.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=59103' = 'object_number=134, tag_tid=8, entry_tid=58738' 2026-02-20T21:10:27.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=135, tag_tid=8, entry_tid=59103' 2026-02-20T21:10:27.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:27.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:27.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:27.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:27.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=59103' 2026-02-20T21:10:27.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:27.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:27.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:27.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:27.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:27.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:27.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=59103' 2026-02-20T21:10:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:27.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:27.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=59103 2026-02-20T21:10:27.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 59103 2026-02-20T21:10:27.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:27.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:27.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:27.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:27.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:27.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:27.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:27.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:27.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:27.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:27.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:27.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:27.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:27.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=59538], [object_number=133, tag_tid=8, entry_tid=59537], [object_number=132, tag_tid=8, entry_tid=59536], [object_number=135, tag_tid=8, entry_tid=59535]]], state=connected] 2026-02-20T21:10:27.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:27.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:27.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:27.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:27.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:27.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:27.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:27.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=59538], [object_number=133, tag_tid=8, entry_tid=59537], [object_number=132, tag_tid=8, entry_tid=59536], [object_number=135, tag_tid=8, entry_tid=59535]]], state=connected] 2026-02-20T21:10:27.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:27.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=134, tag_tid=8, entry_tid=59538' 2026-02-20T21:10:27.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=134, tag_tid=8, entry_tid=59538' 2026-02-20T21:10:27.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=59538' '!=' 'object_number=135, tag_tid=8, entry_tid=59103' 2026-02-20T21:10:27.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:27.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=59538' = 'object_number=135, tag_tid=8, entry_tid=59103' 2026-02-20T21:10:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=134, tag_tid=8, entry_tid=59538' 2026-02-20T21:10:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:27.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:27.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=59538' 2026-02-20T21:10:27.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:27.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:27.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:27.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:27.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:27.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:27.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=59538' 2026-02-20T21:10:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:27.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=59538 2026-02-20T21:10:27.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 59538 2026-02-20T21:10:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:27.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:28.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:28.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:28.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:28.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:28.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:28.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:28.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:28.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:28.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:28.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:28.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:28.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:28.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:28.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:28.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:28.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:28.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:28.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:28.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:28.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=59906], [object_number=133, tag_tid=8, entry_tid=59905], [object_number=132, tag_tid=8, entry_tid=59904], [object_number=135, tag_tid=8, entry_tid=59903]]], state=connected] 2026-02-20T21:10:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:28.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:28.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=59906], [object_number=133, tag_tid=8, entry_tid=59905], [object_number=132, tag_tid=8, entry_tid=59904], [object_number=135, tag_tid=8, entry_tid=59903]]], state=connected] 2026-02-20T21:10:28.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=134, tag_tid=8, entry_tid=59906' 2026-02-20T21:10:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=134, tag_tid=8, entry_tid=59906' 2026-02-20T21:10:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=59906' '!=' 'object_number=134, tag_tid=8, entry_tid=59538' 2026-02-20T21:10:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=59906' = 'object_number=134, tag_tid=8, entry_tid=59538' 2026-02-20T21:10:28.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=134, tag_tid=8, entry_tid=59906' 2026-02-20T21:10:28.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:28.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:28.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:28.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:28.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=59906' 2026-02-20T21:10:28.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:28.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:28.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:28.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:28.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:28.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:28.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=59906' 2026-02-20T21:10:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=59906 2026-02-20T21:10:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 59906 2026-02-20T21:10:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:28.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:28.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:28.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:28.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:28.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:28.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:28.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:28.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:28.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:28.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:28.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:28.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:28.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:28.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:28.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:28.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:28.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:28.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:28.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:28.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:28.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:28.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:28.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:28.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:28.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:28.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:28.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:28.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:28.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:28.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:28.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:28.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:28.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=60262], [object_number=133, tag_tid=8, entry_tid=60261], [object_number=132, tag_tid=8, entry_tid=60260], [object_number=135, tag_tid=8, entry_tid=60259]]], state=connected] 2026-02-20T21:10:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:28.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:28.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:28.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:28.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:28.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:28.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:28.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:28.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:28.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=60262], [object_number=133, tag_tid=8, entry_tid=60261], [object_number=132, tag_tid=8, entry_tid=60260], [object_number=135, tag_tid=8, entry_tid=60259]]], state=connected] 2026-02-20T21:10:28.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=134, tag_tid=8, entry_tid=60262' 2026-02-20T21:10:28.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=134, tag_tid=8, entry_tid=60262' 2026-02-20T21:10:28.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=60262' '!=' 'object_number=134, tag_tid=8, entry_tid=59906' 2026-02-20T21:10:28.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:28.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=60262' = 'object_number=134, tag_tid=8, entry_tid=59906' 2026-02-20T21:10:28.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=134, tag_tid=8, entry_tid=60262' 2026-02-20T21:10:28.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:28.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:28.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:28.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:28.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=60262' 2026-02-20T21:10:28.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:28.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:28.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:28.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:28.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:28.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:28.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:28.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=60262' 2026-02-20T21:10:28.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:28.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=60262 2026-02-20T21:10:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 60262 2026-02-20T21:10:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:28.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:28.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:28.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:28.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:28.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:28.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:28.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:28.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:28.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:28.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:28.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:28.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:28.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:28.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:28.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:28.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:28.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:28.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:28.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:28.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:28.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:28.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:28.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:28.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:28.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:28.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:28.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:28.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:28.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:28.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:28.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:28.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:28.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:28.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:28.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:28.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:28.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:28.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:28.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:28.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:28.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:28.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:28.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:28.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:28.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:28.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=60670], [object_number=133, tag_tid=8, entry_tid=60669], [object_number=132, tag_tid=8, entry_tid=60668], [object_number=135, tag_tid=8, entry_tid=60667]]], state=connected] 2026-02-20T21:10:28.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:28.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:28.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:28.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:28.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:28.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:29.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:29.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:29.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:29.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:29.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=134, tag_tid=8, entry_tid=60670], [object_number=133, tag_tid=8, entry_tid=60669], [object_number=132, tag_tid=8, entry_tid=60668], [object_number=135, tag_tid=8, entry_tid=60667]]], state=connected] 2026-02-20T21:10:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=134, tag_tid=8, entry_tid=60670' 2026-02-20T21:10:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=134, tag_tid=8, entry_tid=60670' 2026-02-20T21:10:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=60670' '!=' 'object_number=134, tag_tid=8, entry_tid=60262' 2026-02-20T21:10:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:29.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=134, tag_tid=8, entry_tid=60670' = 'object_number=134, tag_tid=8, entry_tid=60262' 2026-02-20T21:10:29.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=134, tag_tid=8, entry_tid=60670' 2026-02-20T21:10:29.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:29.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:29.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=60670' 2026-02-20T21:10:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:29.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:29.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:29.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:29.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=134, tag_tid=8, entry_tid=60670' 2026-02-20T21:10:29.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:29.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:29.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=60670 2026-02-20T21:10:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 60670 2026-02-20T21:10:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:29.111 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8363085166501292) > weight (0.25) 2026-02-20T21:10:29.111 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-20T21:10:29.111 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-20T21:10:29.111 INFO:tasks.rbd_mirror_thrash:waiting for 6.138399135311486 secs before reviving daemons 2026-02-20T21:10:29.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:10:29.110+0000 7f275bd45640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 47327) UID: 1000 2026-02-20T21:10:29.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:29.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:29.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:29.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:29.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:29.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:29.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:29.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:29.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:29.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:29.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:29.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:29.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:29.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:29.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:29.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:29.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:29.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:29.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:29.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:29.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:29.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:29.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:29.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:29.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:29.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:29.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:29.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:29.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:29.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:29.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:29.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:29.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802], [object_number=133, tag_tid=8, entry_tid=60801], [object_number=132, tag_tid=8, entry_tid=60800]]], state=connected] 2026-02-20T21:10:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:29.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:29.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:29.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:29.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:29.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:29.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:29.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:29.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:29.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:29.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:29.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:29.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:29.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:29.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802], [object_number=133, tag_tid=8, entry_tid=60801], [object_number=132, tag_tid=8, entry_tid=60800]]], state=connected] 2026-02-20T21:10:29.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=135, tag_tid=8, entry_tid=60803' 2026-02-20T21:10:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=135, tag_tid=8, entry_tid=60803' 2026-02-20T21:10:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=60803' '!=' 'object_number=134, tag_tid=8, entry_tid=60670' 2026-02-20T21:10:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:10:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=60803' = 'object_number=134, tag_tid=8, entry_tid=60670' 2026-02-20T21:10:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=135, tag_tid=8, entry_tid=60803' 2026-02-20T21:10:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=9 2026-02-20T21:10:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=60803' 2026-02-20T21:10:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:10:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=8 2026-02-20T21:10:29.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:29.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:29.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:10:29.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=135, tag_tid=8, entry_tid=60803' 2026-02-20T21:10:29.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:10:29.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:10:29.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=60803 2026-02-20T21:10:29.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 9 = 8 -a 1 -le 60803 2026-02-20T21:10:29.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:10:29.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:29.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:10:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:29.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:29.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:29.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:29.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:29.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:29.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:29.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:29.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:29.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:29.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:29.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:29.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:29.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:29.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:29.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:29.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:29.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802], [object_number=133, tag_tid=8, entry_tid=60801], [object_number=132, tag_tid=8, entry_tid=60800]]], state=connected] 2026-02-20T21:10:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:30.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:30.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:30.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:30.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:30.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802], [object_number=133, tag_tid=8, entry_tid=60801], [object_number=132, tag_tid=8, entry_tid=60800]]], state=connected] 2026-02-20T21:10:30.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=135, tag_tid=8, entry_tid=60803' 2026-02-20T21:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=135, tag_tid=8, entry_tid=60803' 2026-02-20T21:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=135, tag_tid=8, entry_tid=60803' '!=' 'object_number=135, tag_tid=8, entry_tid=60803' 2026-02-20T21:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:10:30.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:10:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:10:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:10:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:10:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:10:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:10:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:10:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:10:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:10:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:10:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:10:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:10:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:10:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:10:30.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:10:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:10:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:10:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:10:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:10:30.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:10:30.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:10:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:10:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:10:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:10:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:10:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:10:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:10:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:10:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:10:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:10:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:10:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:10:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:10:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:10:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:10:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:10:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:10:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:10:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:10:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:10:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:10:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:10:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:10:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:10:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:10:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:10:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:10:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:10:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:10:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:10:30.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:30.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:30.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:10:30.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:10:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:10:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:10:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:10:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:30.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:30.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:10:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:10:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:10:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:10:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:10:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:10:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:10:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:10:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:10:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:30.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:10:30.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:30.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 33 2026-02-20T21:10:30.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 33 2026-02-20T21:10:30.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:10:30.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:30.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=133, tag_tid=9, entry_tid=1], [object_number=132, tag_tid=9, entry_tid=0], [object_number=135, tag_tid=8, entry_tid=60803], [object_number=134, tag_tid=8, entry_tid=60802]]], state=connected] 2026-02-20T21:10:30.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:10:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:30.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=133, tag_tid=9, entry_tid=1' -a 'object_number=133, tag_tid=9, entry_tid=1' = 'object_number=133, tag_tid=9, entry_tid=1' 2026-02-20T21:10:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap3 2026-02-20T21:10:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3 2026-02-20T21:10:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:10:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:10:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:10:31.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap3 2026-02-20T21:10:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-20T21:10:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:10:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:10:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 5 2026-02-20T21:10:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ba958b175d 2026-02-20T21:10:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ba958b175d 2026-02-20T21:10:31.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:10:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:10:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:10:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:10:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:06:40 2026 2026-02-20T21:10:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:09:19 2026 2026-02-20T21:10:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:10:27 2026 2026-02-20T21:10:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:10:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ba958b175d 2026-02-20T21:10:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:10:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:10:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:10:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:10:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:31.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap2-clean ']' 2026-02-20T21:10:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap2-clean 2026-02-20T21:10:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2-clean 2026-02-20T21:10:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:10:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:10:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:10:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:10:31.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:10:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:10:31.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:10:32.008 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.006+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:32.010 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.008+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:10:32.010 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.008+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:10:32.011 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.008+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:10:32.011 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.008+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:10:32.011 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.008+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:10:32.151 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.149+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:10:32.152 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.151+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:32.153 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.151+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:32.153 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.151+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:32.154 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.151+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:32.249 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.247+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:10:32.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.248+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:10:32.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.248+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:10:32.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.248+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:10:32.251 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.248+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:10:32.288 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:32.286+0000 7f4129cc6640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:10:32.288 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:32.286+0000 7f412bcca640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:10:32.289 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.287+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:10:32.289 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:32.287+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:10:32.289 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:32.287+0000 7f41304d3640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:10:32.290 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.288+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:32.290 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.288+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:32.291 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.288+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:32.291 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:32.288+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:32.353 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:32.351+0000 7f412bcca640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:10:32.353 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:32.351+0000 7f412c4cb640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:10:32.354 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:32.352+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:10:32.354 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:32.352+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:10:32.354 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:32.352+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:10:32.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:10:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:10:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:10:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap3 2026-02-20T21:10:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3 2026-02-20T21:10:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:10:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:10:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:10:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:10:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:10:32.857 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7702597169062001) > weight (0.25) 2026-02-20T21:10:32.857 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-20T21:10:32.857 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-20T21:10:32.857 INFO:tasks.rbd_mirror_thrash:waiting for 9.50348177612953 secs before reviving daemons 2026-02-20T21:10:32.858 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:10:32.855+0000 7fcf25efc640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 47116) UID: 1000 2026-02-20T21:10:33.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:10:33.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:10:33.112 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.110+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:10:33.113 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.111+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:10:33.113 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.112+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:10:33.115 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.114+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:33.118 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.116+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:10:33.196 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.194+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:10:33.197 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.194+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:10:33.197 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.195+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:10:33.197 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.195+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:10:33.197 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.195+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:10:33.264 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.263+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:10:33.265 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.264+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:10:33.266 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.264+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:10:33.267 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.265+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:33.267 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.265+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:33.267 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.265+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:33.302 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.301+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:10:33.303 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.301+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:10:33.371 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.370+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:10:33.372 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.370+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:10:33.372 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.370+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:10:33.373 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.370+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:10:33.373 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.370+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:10:33.373 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.370+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:10:33.407 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.405+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:10:33.410 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.407+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:33.410 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.407+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:33.410 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.408+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:33.410 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:10:33.408+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:10:33.483 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:33.481+0000 7f412e4cf640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:10:33.483 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:33.481+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:10:33.483 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:33.481+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:10:33.484 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:33.482+0000 7f412bcca640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:10:33.484 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:10:33.482+0000 7f412ecd0640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:10:33.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:10:33.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:10:33.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:10:33.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap3-clean 2026-02-20T21:10:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap3-clean 2026-02-20T21:10:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:10:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3-clean 2026-02-20T21:10:33.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap3-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:10:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:10:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:10:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:10:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:10:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=5 2026-02-20T21:10:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:10:34.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:10:35.251 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-20T21:10:35.251 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:10:35.251 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-20T21:10:35.251 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-20T21:10:35.252 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-20T21:10:35.255 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-20T21:10:35.255 INFO:tasks.rbd_mirror_thrash:waiting for 60.77038720325382 secs before thrashing 2026-02-20T21:10:35.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 67164160 bytes 2026-02-20T21:10:36.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 151947264 bytes 2026-02-20T21:10:37.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 180855808 bytes 2026-02-20T21:10:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 266065920 bytes 2026-02-20T21:10:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 266682368 bytes 2026-02-20T21:10:38.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 362909696 bytes 2026-02-20T21:10:39.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 384413696 bytes 2026-02-20T21:10:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:10:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:10:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:10:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:10:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:10:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:10:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:10:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:10:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:10:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:10:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:10:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:10:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:10:40.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:10:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:10:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:10:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: down+unknown 2026-02-20T21:10:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: status not found 2026-02-20T21:10:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20T21:10:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:10:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:10:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:10:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:10:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:10:38' 2026-02-20T21:10:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:10:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:10:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:40.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:10:42.361 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-20T21:10:42.362 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:10:42.362 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-20T21:10:42.362 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-20T21:10:42.362 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-20T21:10:42.366 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-20T21:10:42.366 INFO:tasks.rbd_mirror_thrash:waiting for 86.06385330889523 secs before thrashing 2026-02-20T21:10:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:10:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:10:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:10:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:10:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:10:42.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:10:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:10:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:10:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: down+unknown 2026-02-20T21:10:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: status not found 2026-02-20T21:10:42.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20T21:10:42.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:10:42.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:10:42.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:10:42.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:10:42.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:10:38' 2026-02-20T21:10:42.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:10:42.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:10:42.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:42.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-20T21:10:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:10:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:10:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:10:46.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:10:46.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:10:46.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:10:46.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:10:46.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:10:46.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: down+unknown 2026-02-20T21:10:46.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: status not found 2026-02-20T21:10:46.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20T21:10:46.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:10:46.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:10:46.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:10:46.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:10:46.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:10:38' 2026-02-20T21:10:46.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:10:46.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:10:46.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:46.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 8 2026-02-20T21:10:54.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:10:54.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:10:54.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:10:54.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:10:54.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:10:54.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:10:54.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:10:54.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:10:54.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:10:54.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:10:54.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:10:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:10:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: down+unknown 2026-02-20T21:10:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: status not found 2026-02-20T21:10:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20T21:10:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:10:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:10:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:10:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:10:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:10:38' 2026-02-20T21:10:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:10:54.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:10:54.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:10:54.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 8 2026-02-20T21:11:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:11:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:11:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:11:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:11:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:11:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:11:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:11:02.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:11:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: down+unknown 2026-02-20T21:11:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: status not found 2026-02-20T21:11:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20T21:11:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:11:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:11:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:11:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:11:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:10:38' 2026-02-20T21:11:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:11:02.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:11:02.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:02.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 8 2026-02-20T21:11:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:11:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:11:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:11:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:11:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:11:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:11:10.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:11:10.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:11:10.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-20T21:11:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":133,"tag_tid":9},"primary_position":{"entry_tid":1,"object_number":133,"tag_tid":10}} 2026-02-20T21:11:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.0 on trial163 2026-02-20T21:11:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:11:08 2026-02-20T21:11:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:11:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:11:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:11:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:11:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:11:08' 2026-02-20T21:11:10.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:11:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-20T21:11:10.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-20T21:11:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":133,"tag_tid":9},"primary_position":{"entry_tid":1,"object_number":133,"tag_tid":10}} 2026-02-20T21:11:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:11:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:11:10.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-20T21:11:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:11:10.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.0 on trial163 2026-02-20T21:11:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-20T21:11:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:11:08' 2026-02-20T21:11:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:11:08' 2026-02-20T21:11:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-20 21:11:08' 2026-02-20T21:11:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:11:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:11:10.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142044287mirror.0trial16320.2.0-677-gf7870454trueOK43304332mirror.2trial16320.2.0-677-gf7870454falseOK43484351mirror.3trial16320.2.0-677-gf7870454falseOK43884391mirror.1trial16320.2.0-677-gf7870454falseOKtestba923525-490d-4a76-8e5f-b742509a94daup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":133,"tag_tid":9},"primary_position":{"entry_tid":1,"object_number":133,"tag_tid":10}}42044287mirror.0trial1632026-02-20 21:11:08cluster28b968eaf-6c24-469f-bea5-c8acb780219eup+stoppedlocal image is primary2026-02-20 21:11:08' 2026-02-20T21:11:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-20T21:11:11.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:11.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:11:08' 2026-02-20T21:11:11.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-20T21:11:11.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:11:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-20T21:11:11.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:11:11.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:11.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:11.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap4 2026-02-20T21:11:11.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap4 2026-02-20T21:11:11.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:11:11.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:11.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:11.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4 2026-02-20T21:11:11.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:11:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:11:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:11:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:11:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:11:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:11:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:11:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:11:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:11:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:11:15.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:15.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:11:15.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:11:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:11:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:11:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:11:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:11:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:11:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:11:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:11:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:11:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:11:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:11:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:11:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:11:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:11:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:11:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:11:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:11:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:11:15.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:11:15.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:11:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:11:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:11:15.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:11:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:11:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:11:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:11:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:11:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:11:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:11:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:11:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:11:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:11:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:11:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:11:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:11:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:11:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:11:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:11:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:16.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:16.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:16.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:16.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:16.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:16.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:16.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=144, tag_tid=11, entry_tid=9500], [object_number=147, tag_tid=11, entry_tid=9499], [object_number=146, tag_tid=11, entry_tid=9498], [object_number=145, tag_tid=11, entry_tid=9497]]], state=connected] 2026-02-20T21:11:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:16.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:16.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=144, tag_tid=11, entry_tid=9500], [object_number=147, tag_tid=11, entry_tid=9499], [object_number=146, tag_tid=11, entry_tid=9498], [object_number=145, tag_tid=11, entry_tid=9497]]], state=connected] 2026-02-20T21:11:16.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=144, tag_tid=11, entry_tid=9500' 2026-02-20T21:11:16.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=144, tag_tid=11, entry_tid=9500' 2026-02-20T21:11:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=144, tag_tid=11, entry_tid=9500' '!=' '' 2026-02-20T21:11:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=144, tag_tid=11, entry_tid=9500' = '' 2026-02-20T21:11:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=144, tag_tid=11, entry_tid=9500' 2026-02-20T21:11:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:16.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:16.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=144, tag_tid=11, entry_tid=9500' 2026-02-20T21:11:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:16.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:16.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=144, tag_tid=11, entry_tid=9500' 2026-02-20T21:11:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=9500 2026-02-20T21:11:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 9500 2026-02-20T21:11:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:16.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:16.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:16.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:16.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:16.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:16.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:16.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:16.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:16.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:16.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:16.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:16.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:16.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:16.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:16.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=145, tag_tid=11, entry_tid=9797], [object_number=144, tag_tid=11, entry_tid=9796], [object_number=147, tag_tid=11, entry_tid=9795], [object_number=146, tag_tid=11, entry_tid=9794]]], state=connected] 2026-02-20T21:11:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=145, tag_tid=11, entry_tid=9797], [object_number=144, tag_tid=11, entry_tid=9796], [object_number=147, tag_tid=11, entry_tid=9795], [object_number=146, tag_tid=11, entry_tid=9794]]], state=connected] 2026-02-20T21:11:16.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=145, tag_tid=11, entry_tid=9797' 2026-02-20T21:11:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=145, tag_tid=11, entry_tid=9797' 2026-02-20T21:11:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=11, entry_tid=9797' '!=' 'object_number=144, tag_tid=11, entry_tid=9500' 2026-02-20T21:11:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=11, entry_tid=9797' = 'object_number=144, tag_tid=11, entry_tid=9500' 2026-02-20T21:11:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=145, tag_tid=11, entry_tid=9797' 2026-02-20T21:11:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=11, entry_tid=9797' 2026-02-20T21:11:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:16.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:16.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:16.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=11, entry_tid=9797' 2026-02-20T21:11:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=9797 2026-02-20T21:11:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 9797 2026-02-20T21:11:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:16.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:16.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:16.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:16.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:16.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:16.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:16.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:16.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:16.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:16.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:16.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:16.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:16.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:16.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:16.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:16.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:16.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=144, tag_tid=11, entry_tid=10112], [object_number=147, tag_tid=11, entry_tid=10111], [object_number=146, tag_tid=11, entry_tid=10110], [object_number=145, tag_tid=11, entry_tid=10109]]], state=connected] 2026-02-20T21:11:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:16.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:16.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=144, tag_tid=11, entry_tid=10112], [object_number=147, tag_tid=11, entry_tid=10111], [object_number=146, tag_tid=11, entry_tid=10110], [object_number=145, tag_tid=11, entry_tid=10109]]], state=connected] 2026-02-20T21:11:16.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=144, tag_tid=11, entry_tid=10112' 2026-02-20T21:11:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=144, tag_tid=11, entry_tid=10112' 2026-02-20T21:11:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=144, tag_tid=11, entry_tid=10112' '!=' 'object_number=145, tag_tid=11, entry_tid=9797' 2026-02-20T21:11:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=144, tag_tid=11, entry_tid=10112' = 'object_number=145, tag_tid=11, entry_tid=9797' 2026-02-20T21:11:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=144, tag_tid=11, entry_tid=10112' 2026-02-20T21:11:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:16.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=144, tag_tid=11, entry_tid=10112' 2026-02-20T21:11:16.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:16.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=144, tag_tid=11, entry_tid=10112' 2026-02-20T21:11:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:16.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=10112 2026-02-20T21:11:16.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 10112 2026-02-20T21:11:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:17.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:17.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:17.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:17.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:17.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:17.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:17.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:17.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:17.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:17.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:17.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:17.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:17.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:17.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=146, tag_tid=11, entry_tid=10454], [object_number=145, tag_tid=11, entry_tid=10453], [object_number=144, tag_tid=11, entry_tid=10452], [object_number=147, tag_tid=11, entry_tid=10451]]], state=connected] 2026-02-20T21:11:17.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:17.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:17.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:17.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:17.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=146, tag_tid=11, entry_tid=10454], [object_number=145, tag_tid=11, entry_tid=10453], [object_number=144, tag_tid=11, entry_tid=10452], [object_number=147, tag_tid=11, entry_tid=10451]]], state=connected] 2026-02-20T21:11:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=146, tag_tid=11, entry_tid=10454' 2026-02-20T21:11:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=146, tag_tid=11, entry_tid=10454' 2026-02-20T21:11:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=11, entry_tid=10454' '!=' 'object_number=144, tag_tid=11, entry_tid=10112' 2026-02-20T21:11:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=11, entry_tid=10454' = 'object_number=144, tag_tid=11, entry_tid=10112' 2026-02-20T21:11:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=146, tag_tid=11, entry_tid=10454' 2026-02-20T21:11:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=11, entry_tid=10454' 2026-02-20T21:11:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=11, entry_tid=10454' 2026-02-20T21:11:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=10454 2026-02-20T21:11:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 10454 2026-02-20T21:11:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:17.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:17.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:17.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:17.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:17.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:17.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:17.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:17.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:17.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:17.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=147, tag_tid=11, entry_tid=10827], [object_number=146, tag_tid=11, entry_tid=10826], [object_number=145, tag_tid=11, entry_tid=10825], [object_number=144, tag_tid=11, entry_tid=10824]]], state=connected] 2026-02-20T21:11:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:17.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=147, tag_tid=11, entry_tid=10827], [object_number=146, tag_tid=11, entry_tid=10826], [object_number=145, tag_tid=11, entry_tid=10825], [object_number=144, tag_tid=11, entry_tid=10824]]], state=connected] 2026-02-20T21:11:17.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=147, tag_tid=11, entry_tid=10827' 2026-02-20T21:11:17.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=147, tag_tid=11, entry_tid=10827' 2026-02-20T21:11:17.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=147, tag_tid=11, entry_tid=10827' '!=' 'object_number=146, tag_tid=11, entry_tid=10454' 2026-02-20T21:11:17.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:17.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=147, tag_tid=11, entry_tid=10827' = 'object_number=146, tag_tid=11, entry_tid=10454' 2026-02-20T21:11:17.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=147, tag_tid=11, entry_tid=10827' 2026-02-20T21:11:17.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:17.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:17.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:17.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:17.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=147, tag_tid=11, entry_tid=10827' 2026-02-20T21:11:17.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:17.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:17.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:17.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:17.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:17.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:17.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:17.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=147, tag_tid=11, entry_tid=10827' 2026-02-20T21:11:17.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:17.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:17.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=10827 2026-02-20T21:11:17.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 10827 2026-02-20T21:11:17.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:17.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:17.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:17.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:17.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:17.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:17.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:17.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:17.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:17.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=147, tag_tid=11, entry_tid=11155], [object_number=146, tag_tid=11, entry_tid=11154], [object_number=145, tag_tid=11, entry_tid=11153], [object_number=144, tag_tid=11, entry_tid=11152]]], state=connected] 2026-02-20T21:11:17.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:17.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:17.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=147, tag_tid=11, entry_tid=11155], [object_number=146, tag_tid=11, entry_tid=11154], [object_number=145, tag_tid=11, entry_tid=11153], [object_number=144, tag_tid=11, entry_tid=11152]]], state=connected] 2026-02-20T21:11:17.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=147, tag_tid=11, entry_tid=11155' 2026-02-20T21:11:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=147, tag_tid=11, entry_tid=11155' 2026-02-20T21:11:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=147, tag_tid=11, entry_tid=11155' '!=' 'object_number=147, tag_tid=11, entry_tid=10827' 2026-02-20T21:11:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=147, tag_tid=11, entry_tid=11155' = 'object_number=147, tag_tid=11, entry_tid=10827' 2026-02-20T21:11:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=147, tag_tid=11, entry_tid=11155' 2026-02-20T21:11:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=147, tag_tid=11, entry_tid=11155' 2026-02-20T21:11:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=147, tag_tid=11, entry_tid=11155' 2026-02-20T21:11:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:17.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:17.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=11155 2026-02-20T21:11:17.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 11155 2026-02-20T21:11:17.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:17.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:17.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:18.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:18.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:18.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:18.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:18.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:18.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:18.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=147, tag_tid=11, entry_tid=11487], [object_number=146, tag_tid=11, entry_tid=11486], [object_number=145, tag_tid=11, entry_tid=11485], [object_number=144, tag_tid=11, entry_tid=11484]]], state=connected] 2026-02-20T21:11:18.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:18.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:18.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:18.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=147, tag_tid=11, entry_tid=11487], [object_number=146, tag_tid=11, entry_tid=11486], [object_number=145, tag_tid=11, entry_tid=11485], [object_number=144, tag_tid=11, entry_tid=11484]]], state=connected] 2026-02-20T21:11:18.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=147, tag_tid=11, entry_tid=11487' 2026-02-20T21:11:18.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=147, tag_tid=11, entry_tid=11487' 2026-02-20T21:11:18.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=147, tag_tid=11, entry_tid=11487' '!=' 'object_number=147, tag_tid=11, entry_tid=11155' 2026-02-20T21:11:18.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:18.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=147, tag_tid=11, entry_tid=11487' = 'object_number=147, tag_tid=11, entry_tid=11155' 2026-02-20T21:11:18.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=147, tag_tid=11, entry_tid=11487' 2026-02-20T21:11:18.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:18.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:18.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:18.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=147, tag_tid=11, entry_tid=11487' 2026-02-20T21:11:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:18.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:18.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=147, tag_tid=11, entry_tid=11487' 2026-02-20T21:11:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:18.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=11487 2026-02-20T21:11:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 11487 2026-02-20T21:11:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:18.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:18.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:18.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:18.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:18.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:18.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:18.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:18.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:18.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:18.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:18.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:18.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:18.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:18.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:18.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:18.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:18.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:18.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:18.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:18.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:18.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:18.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:18.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:18.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:18.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:18.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:18.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:18.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:18.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:18.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:18.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:18.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:18.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:18.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:18.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:18.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:18.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:18.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:18.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:18.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:18.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:18.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:18.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:18.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:18.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:18.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:18.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:18.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:18.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:18.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:18.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:18.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=145, tag_tid=11, entry_tid=11841], [object_number=144, tag_tid=11, entry_tid=11840], [object_number=147, tag_tid=11, entry_tid=11839], [object_number=146, tag_tid=11, entry_tid=11838]]], state=connected] 2026-02-20T21:11:18.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:18.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:18.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:18.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:18.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:18.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:18.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=145, tag_tid=11, entry_tid=11841], [object_number=144, tag_tid=11, entry_tid=11840], [object_number=147, tag_tid=11, entry_tid=11839], [object_number=146, tag_tid=11, entry_tid=11838]]], state=connected] 2026-02-20T21:11:18.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=145, tag_tid=11, entry_tid=11841' 2026-02-20T21:11:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=145, tag_tid=11, entry_tid=11841' 2026-02-20T21:11:18.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=11, entry_tid=11841' '!=' 'object_number=147, tag_tid=11, entry_tid=11487' 2026-02-20T21:11:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=145, tag_tid=11, entry_tid=11841' = 'object_number=147, tag_tid=11, entry_tid=11487' 2026-02-20T21:11:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=145, tag_tid=11, entry_tid=11841' 2026-02-20T21:11:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:18.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=11, entry_tid=11841' 2026-02-20T21:11:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=145, tag_tid=11, entry_tid=11841' 2026-02-20T21:11:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=11841 2026-02-20T21:11:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 11841 2026-02-20T21:11:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:18.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:18.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:18.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:18.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:18.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:18.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:18.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:18.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:18.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:18.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:18.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:18.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:18.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:18.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:18.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:18.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:18.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:18.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=146, tag_tid=11, entry_tid=12190], [object_number=145, tag_tid=11, entry_tid=12189], [object_number=144, tag_tid=11, entry_tid=12188], [object_number=147, tag_tid=11, entry_tid=12187]]], state=connected] 2026-02-20T21:11:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:18.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:18.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:18.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:18.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:18.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:18.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:18.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 36 2026-02-20T21:11:19.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:19.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:19.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:19.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=146, tag_tid=11, entry_tid=12190], [object_number=145, tag_tid=11, entry_tid=12189], [object_number=144, tag_tid=11, entry_tid=12188], [object_number=147, tag_tid=11, entry_tid=12187]]], state=connected] 2026-02-20T21:11:19.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=146, tag_tid=11, entry_tid=12190' 2026-02-20T21:11:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=146, tag_tid=11, entry_tid=12190' 2026-02-20T21:11:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=11, entry_tid=12190' '!=' 'object_number=145, tag_tid=11, entry_tid=11841' 2026-02-20T21:11:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=146, tag_tid=11, entry_tid=12190' = 'object_number=145, tag_tid=11, entry_tid=11841' 2026-02-20T21:11:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=146, tag_tid=11, entry_tid=12190' 2026-02-20T21:11:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=11, entry_tid=12190' 2026-02-20T21:11:19.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:19.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:19.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:19.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:19.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:19.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=146, tag_tid=11, entry_tid=12190' 2026-02-20T21:11:19.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:19.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:19.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=12190 2026-02-20T21:11:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 12190 2026-02-20T21:11:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:19.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:19.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:19.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:19.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:19.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:19.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:19.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:19.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:19.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:19.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=151, tag_tid=11, entry_tid=12651], [object_number=150, tag_tid=11, entry_tid=12650], [object_number=149, tag_tid=11, entry_tid=12649], [object_number=148, tag_tid=11, entry_tid=12648]]], state=connected] 2026-02-20T21:11:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:19.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:19.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:19.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=151, tag_tid=11, entry_tid=12651], [object_number=150, tag_tid=11, entry_tid=12650], [object_number=149, tag_tid=11, entry_tid=12649], [object_number=148, tag_tid=11, entry_tid=12648]]], state=connected] 2026-02-20T21:11:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=151, tag_tid=11, entry_tid=12651' 2026-02-20T21:11:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=151, tag_tid=11, entry_tid=12651' 2026-02-20T21:11:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=151, tag_tid=11, entry_tid=12651' '!=' 'object_number=146, tag_tid=11, entry_tid=12190' 2026-02-20T21:11:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=151, tag_tid=11, entry_tid=12651' = 'object_number=146, tag_tid=11, entry_tid=12190' 2026-02-20T21:11:19.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=151, tag_tid=11, entry_tid=12651' 2026-02-20T21:11:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:19.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=151, tag_tid=11, entry_tid=12651' 2026-02-20T21:11:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:19.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:19.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:19.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=151, tag_tid=11, entry_tid=12651' 2026-02-20T21:11:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:19.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=12651 2026-02-20T21:11:19.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 12651 2026-02-20T21:11:19.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:19.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:19.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:19.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:19.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:19.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:19.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:19.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:19.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:19.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:19.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:19.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:19.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:19.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:19.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:19.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:19.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:19.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:19.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:19.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:19.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:19.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:19.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:19.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:19.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:19.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:19.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:19.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:19.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:19.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:19.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:19.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:19.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:19.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=149, tag_tid=11, entry_tid=12961], [object_number=148, tag_tid=11, entry_tid=12960], [object_number=151, tag_tid=11, entry_tid=12959], [object_number=150, tag_tid=11, entry_tid=12958]]], state=connected] 2026-02-20T21:11:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:19.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:19.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:19.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:19.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=149, tag_tid=11, entry_tid=12961], [object_number=148, tag_tid=11, entry_tid=12960], [object_number=151, tag_tid=11, entry_tid=12959], [object_number=150, tag_tid=11, entry_tid=12958]]], state=connected] 2026-02-20T21:11:19.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:19.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=149, tag_tid=11, entry_tid=12961' 2026-02-20T21:11:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=149, tag_tid=11, entry_tid=12961' 2026-02-20T21:11:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=11, entry_tid=12961' '!=' 'object_number=151, tag_tid=11, entry_tid=12651' 2026-02-20T21:11:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=11, entry_tid=12961' = 'object_number=151, tag_tid=11, entry_tid=12651' 2026-02-20T21:11:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=149, tag_tid=11, entry_tid=12961' 2026-02-20T21:11:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=11, entry_tid=12961' 2026-02-20T21:11:19.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:19.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:19.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:19.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:19.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=11, entry_tid=12961' 2026-02-20T21:11:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:19.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=12961 2026-02-20T21:11:19.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 12961 2026-02-20T21:11:19.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:19.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:19.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:19.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:19.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:19.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:19.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:19.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:19.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:19.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:19.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:20.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=148, tag_tid=11, entry_tid=13304], [object_number=151, tag_tid=11, entry_tid=13303], [object_number=150, tag_tid=11, entry_tid=13302], [object_number=149, tag_tid=11, entry_tid=13301]]], state=connected] 2026-02-20T21:11:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:20.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:20.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:20.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=148, tag_tid=11, entry_tid=13304], [object_number=151, tag_tid=11, entry_tid=13303], [object_number=150, tag_tid=11, entry_tid=13302], [object_number=149, tag_tid=11, entry_tid=13301]]], state=connected] 2026-02-20T21:11:20.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=148, tag_tid=11, entry_tid=13304' 2026-02-20T21:11:20.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=148, tag_tid=11, entry_tid=13304' 2026-02-20T21:11:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=148, tag_tid=11, entry_tid=13304' '!=' 'object_number=149, tag_tid=11, entry_tid=12961' 2026-02-20T21:11:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=148, tag_tid=11, entry_tid=13304' = 'object_number=149, tag_tid=11, entry_tid=12961' 2026-02-20T21:11:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=148, tag_tid=11, entry_tid=13304' 2026-02-20T21:11:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:20.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=148, tag_tid=11, entry_tid=13304' 2026-02-20T21:11:20.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:20.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:20.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:20.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:20.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:20.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:20.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:20.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=148, tag_tid=11, entry_tid=13304' 2026-02-20T21:11:20.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:20.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=13304 2026-02-20T21:11:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 13304 2026-02-20T21:11:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:20.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:20.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:20.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:20.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:20.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:20.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:20.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:20.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:20.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=150, tag_tid=11, entry_tid=13614], [object_number=149, tag_tid=11, entry_tid=13613], [object_number=148, tag_tid=11, entry_tid=13612], [object_number=151, tag_tid=11, entry_tid=13611]]], state=connected] 2026-02-20T21:11:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:20.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:20.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:20.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:20.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:20.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:20.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:20.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:20.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:20.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:20.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:20.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:20.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:20.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:20.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:20.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:20.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=150, tag_tid=11, entry_tid=13614], [object_number=149, tag_tid=11, entry_tid=13613], [object_number=148, tag_tid=11, entry_tid=13612], [object_number=151, tag_tid=11, entry_tid=13611]]], state=connected] 2026-02-20T21:11:20.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=150, tag_tid=11, entry_tid=13614' 2026-02-20T21:11:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=150, tag_tid=11, entry_tid=13614' 2026-02-20T21:11:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=11, entry_tid=13614' '!=' 'object_number=148, tag_tid=11, entry_tid=13304' 2026-02-20T21:11:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=11, entry_tid=13614' = 'object_number=148, tag_tid=11, entry_tid=13304' 2026-02-20T21:11:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=150, tag_tid=11, entry_tid=13614' 2026-02-20T21:11:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:20.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:20.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=11, entry_tid=13614' 2026-02-20T21:11:20.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:20.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:20.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:20.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:20.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:20.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:20.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:20.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=11, entry_tid=13614' 2026-02-20T21:11:20.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:20.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=13614 2026-02-20T21:11:20.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 13614 2026-02-20T21:11:20.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:20.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:20.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:20.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:20.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:20.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:20.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:20.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:20.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:20.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:20.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:20.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:20.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:20.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:20.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:20.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:20.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:20.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:20.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:20.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:20.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:20.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:20.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:20.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:20.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:20.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:20.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:20.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:20.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:20.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:20.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:20.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:20.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:20.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:20.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=150, tag_tid=11, entry_tid=13914], [object_number=149, tag_tid=11, entry_tid=13913], [object_number=148, tag_tid=11, entry_tid=13912], [object_number=151, tag_tid=11, entry_tid=13911]]], state=connected] 2026-02-20T21:11:20.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:20.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:20.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:20.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:20.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=150, tag_tid=11, entry_tid=13914], [object_number=149, tag_tid=11, entry_tid=13913], [object_number=148, tag_tid=11, entry_tid=13912], [object_number=151, tag_tid=11, entry_tid=13911]]], state=connected] 2026-02-20T21:11:20.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:20.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=150, tag_tid=11, entry_tid=13914' 2026-02-20T21:11:20.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=150, tag_tid=11, entry_tid=13914' 2026-02-20T21:11:20.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=11, entry_tid=13914' '!=' 'object_number=150, tag_tid=11, entry_tid=13614' 2026-02-20T21:11:20.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:20.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=11, entry_tid=13914' = 'object_number=150, tag_tid=11, entry_tid=13614' 2026-02-20T21:11:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=150, tag_tid=11, entry_tid=13914' 2026-02-20T21:11:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:20.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:20.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:20.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=11, entry_tid=13914' 2026-02-20T21:11:20.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:20.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:20.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:20.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:20.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:20.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:20.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:20.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=11, entry_tid=13914' 2026-02-20T21:11:20.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:20.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=13914 2026-02-20T21:11:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 13914 2026-02-20T21:11:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:20.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:20.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:20.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:20.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:20.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:20.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:20.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:20.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:20.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:20.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:20.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:20.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:20.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:20.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:20.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:20.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:20.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:20.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:20.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:20.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:20.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:20.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:20.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:20.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:20.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:20.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:20.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:20.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:20.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:20.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:20.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:20.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:21.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=149, tag_tid=11, entry_tid=14233], [object_number=148, tag_tid=11, entry_tid=14232], [object_number=151, tag_tid=11, entry_tid=14231], [object_number=150, tag_tid=11, entry_tid=14230]]], state=connected] 2026-02-20T21:11:21.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:21.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:21.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:21.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:21.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=149, tag_tid=11, entry_tid=14233], [object_number=148, tag_tid=11, entry_tid=14232], [object_number=151, tag_tid=11, entry_tid=14231], [object_number=150, tag_tid=11, entry_tid=14230]]], state=connected] 2026-02-20T21:11:21.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=149, tag_tid=11, entry_tid=14233' 2026-02-20T21:11:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=149, tag_tid=11, entry_tid=14233' 2026-02-20T21:11:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=11, entry_tid=14233' '!=' 'object_number=150, tag_tid=11, entry_tid=13914' 2026-02-20T21:11:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=11, entry_tid=14233' = 'object_number=150, tag_tid=11, entry_tid=13914' 2026-02-20T21:11:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=149, tag_tid=11, entry_tid=14233' 2026-02-20T21:11:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:21.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:21.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=11, entry_tid=14233' 2026-02-20T21:11:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:21.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:21.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:21.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=11, entry_tid=14233' 2026-02-20T21:11:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:21.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:21.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=14233 2026-02-20T21:11:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 14233 2026-02-20T21:11:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:21.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:21.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:21.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:21.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:21.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:21.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:21.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:21.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:21.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:21.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:21.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:21.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:21.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:21.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:21.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:21.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:21.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:21.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=148, tag_tid=11, entry_tid=14576], [object_number=151, tag_tid=11, entry_tid=14575], [object_number=150, tag_tid=11, entry_tid=14574], [object_number=149, tag_tid=11, entry_tid=14573]]], state=connected] 2026-02-20T21:11:21.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:21.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:21.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:21.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:21.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:21.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:21.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=148, tag_tid=11, entry_tid=14576], [object_number=151, tag_tid=11, entry_tid=14575], [object_number=150, tag_tid=11, entry_tid=14574], [object_number=149, tag_tid=11, entry_tid=14573]]], state=connected] 2026-02-20T21:11:21.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=148, tag_tid=11, entry_tid=14576' 2026-02-20T21:11:21.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=148, tag_tid=11, entry_tid=14576' 2026-02-20T21:11:21.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=148, tag_tid=11, entry_tid=14576' '!=' 'object_number=149, tag_tid=11, entry_tid=14233' 2026-02-20T21:11:21.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:21.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=148, tag_tid=11, entry_tid=14576' = 'object_number=149, tag_tid=11, entry_tid=14233' 2026-02-20T21:11:21.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=148, tag_tid=11, entry_tid=14576' 2026-02-20T21:11:21.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:21.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:21.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:21.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:21.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=148, tag_tid=11, entry_tid=14576' 2026-02-20T21:11:21.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:21.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:21.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:21.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:21.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:21.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:21.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:21.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=148, tag_tid=11, entry_tid=14576' 2026-02-20T21:11:21.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:21.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=14576 2026-02-20T21:11:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 14576 2026-02-20T21:11:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:21.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:21.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:21.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:21.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:21.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:21.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:21.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:21.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:21.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:21.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:21.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:21.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:21.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=151, tag_tid=11, entry_tid=14899], [object_number=150, tag_tid=11, entry_tid=14898], [object_number=149, tag_tid=11, entry_tid=14897], [object_number=148, tag_tid=11, entry_tid=14896]]], state=connected] 2026-02-20T21:11:21.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:21.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:21.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:21.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:21.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:21.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:21.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:21.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:21.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:21.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:21.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:21.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:21.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:21.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:21.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=151, tag_tid=11, entry_tid=14899], [object_number=150, tag_tid=11, entry_tid=14898], [object_number=149, tag_tid=11, entry_tid=14897], [object_number=148, tag_tid=11, entry_tid=14896]]], state=connected] 2026-02-20T21:11:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:21.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=151, tag_tid=11, entry_tid=14899' 2026-02-20T21:11:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=151, tag_tid=11, entry_tid=14899' 2026-02-20T21:11:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=151, tag_tid=11, entry_tid=14899' '!=' 'object_number=148, tag_tid=11, entry_tid=14576' 2026-02-20T21:11:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=151, tag_tid=11, entry_tid=14899' = 'object_number=148, tag_tid=11, entry_tid=14576' 2026-02-20T21:11:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=151, tag_tid=11, entry_tid=14899' 2026-02-20T21:11:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:21.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=151, tag_tid=11, entry_tid=14899' 2026-02-20T21:11:21.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:21.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:21.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:21.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:21.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:21.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=151, tag_tid=11, entry_tid=14899' 2026-02-20T21:11:21.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:21.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=14899 2026-02-20T21:11:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 14899 2026-02-20T21:11:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:21.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:21.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:21.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:21.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:21.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:21.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:21.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:21.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:21.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:21.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:22.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:22.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:22.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:22.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:22.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:22.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=149, tag_tid=11, entry_tid=15217], [object_number=148, tag_tid=11, entry_tid=15216], [object_number=151, tag_tid=11, entry_tid=15215], [object_number=150, tag_tid=11, entry_tid=15214]]], state=connected] 2026-02-20T21:11:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:22.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:22.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:22.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:22.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:22.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=149, tag_tid=11, entry_tid=15217], [object_number=148, tag_tid=11, entry_tid=15216], [object_number=151, tag_tid=11, entry_tid=15215], [object_number=150, tag_tid=11, entry_tid=15214]]], state=connected] 2026-02-20T21:11:22.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=149, tag_tid=11, entry_tid=15217' 2026-02-20T21:11:22.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=149, tag_tid=11, entry_tid=15217' 2026-02-20T21:11:22.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=11, entry_tid=15217' '!=' 'object_number=151, tag_tid=11, entry_tid=14899' 2026-02-20T21:11:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=149, tag_tid=11, entry_tid=15217' = 'object_number=151, tag_tid=11, entry_tid=14899' 2026-02-20T21:11:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=149, tag_tid=11, entry_tid=15217' 2026-02-20T21:11:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:22.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:22.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:22.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=11, entry_tid=15217' 2026-02-20T21:11:22.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:22.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:22.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:22.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:22.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=149, tag_tid=11, entry_tid=15217' 2026-02-20T21:11:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=15217 2026-02-20T21:11:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 15217 2026-02-20T21:11:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:22.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:22.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:22.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:22.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:22.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:22.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:22.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:22.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:22.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:22.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:22.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:22.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:22.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:22.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:22.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:22.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:22.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:22.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:22.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:22.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:22.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:22.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:22.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:22.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:22.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:22.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:22.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:22.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:22.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:22.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=148, tag_tid=11, entry_tid=15756], [object_number=151, tag_tid=11, entry_tid=15755], [object_number=150, tag_tid=11, entry_tid=15754], [object_number=149, tag_tid=11, entry_tid=15753]]], state=connected] 2026-02-20T21:11:22.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:22.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:22.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:22.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:22.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:22.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=148, tag_tid=11, entry_tid=15756], [object_number=151, tag_tid=11, entry_tid=15755], [object_number=150, tag_tid=11, entry_tid=15754], [object_number=149, tag_tid=11, entry_tid=15753]]], state=connected] 2026-02-20T21:11:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=148, tag_tid=11, entry_tid=15756' 2026-02-20T21:11:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=148, tag_tid=11, entry_tid=15756' 2026-02-20T21:11:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=148, tag_tid=11, entry_tid=15756' '!=' 'object_number=149, tag_tid=11, entry_tid=15217' 2026-02-20T21:11:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=148, tag_tid=11, entry_tid=15756' = 'object_number=149, tag_tid=11, entry_tid=15217' 2026-02-20T21:11:22.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=148, tag_tid=11, entry_tid=15756' 2026-02-20T21:11:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=148, tag_tid=11, entry_tid=15756' 2026-02-20T21:11:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:22.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:22.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:22.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:22.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=148, tag_tid=11, entry_tid=15756' 2026-02-20T21:11:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:22.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=15756 2026-02-20T21:11:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 15756 2026-02-20T21:11:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:22.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:22.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:22.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:22.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:22.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:22.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:22.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:22.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:22.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:22.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:22.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:22.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:22.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:22.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:22.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:22.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:22.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:22.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:22.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:22.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:22.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:22.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:22.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:22.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:22.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:22.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:22.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=150, tag_tid=11, entry_tid=16134], [object_number=149, tag_tid=11, entry_tid=16133], [object_number=148, tag_tid=11, entry_tid=16132], [object_number=151, tag_tid=11, entry_tid=16131]]], state=connected] 2026-02-20T21:11:22.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:22.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:22.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:22.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:22.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:22.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:22.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:22.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:22.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=150, tag_tid=11, entry_tid=16134], [object_number=149, tag_tid=11, entry_tid=16133], [object_number=148, tag_tid=11, entry_tid=16132], [object_number=151, tag_tid=11, entry_tid=16131]]], state=connected] 2026-02-20T21:11:22.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=150, tag_tid=11, entry_tid=16134' 2026-02-20T21:11:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=150, tag_tid=11, entry_tid=16134' 2026-02-20T21:11:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=11, entry_tid=16134' '!=' 'object_number=148, tag_tid=11, entry_tid=15756' 2026-02-20T21:11:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=150, tag_tid=11, entry_tid=16134' = 'object_number=148, tag_tid=11, entry_tid=15756' 2026-02-20T21:11:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=150, tag_tid=11, entry_tid=16134' 2026-02-20T21:11:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=11, entry_tid=16134' 2026-02-20T21:11:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:22.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:22.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:22.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:22.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=150, tag_tid=11, entry_tid=16134' 2026-02-20T21:11:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:22.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:22.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=16134 2026-02-20T21:11:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 16134 2026-02-20T21:11:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:22.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:23.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:23.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:23.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:23.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:23.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:23.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:23.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:23.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:23.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:23.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:23.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:23.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:23.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:23.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:23.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:23.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:23.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:23.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:23.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=151, tag_tid=11, entry_tid=16495], [object_number=150, tag_tid=11, entry_tid=16494], [object_number=149, tag_tid=11, entry_tid=16493], [object_number=148, tag_tid=11, entry_tid=16492]]], state=connected] 2026-02-20T21:11:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:23.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:23.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:23.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:23.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:23.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 37 2026-02-20T21:11:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=151, tag_tid=11, entry_tid=16495], [object_number=150, tag_tid=11, entry_tid=16494], [object_number=149, tag_tid=11, entry_tid=16493], [object_number=148, tag_tid=11, entry_tid=16492]]], state=connected] 2026-02-20T21:11:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=151, tag_tid=11, entry_tid=16495' 2026-02-20T21:11:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=151, tag_tid=11, entry_tid=16495' 2026-02-20T21:11:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=151, tag_tid=11, entry_tid=16495' '!=' 'object_number=150, tag_tid=11, entry_tid=16134' 2026-02-20T21:11:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=151, tag_tid=11, entry_tid=16495' = 'object_number=150, tag_tid=11, entry_tid=16134' 2026-02-20T21:11:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=151, tag_tid=11, entry_tid=16495' 2026-02-20T21:11:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=151, tag_tid=11, entry_tid=16495' 2026-02-20T21:11:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:23.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:23.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:23.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:23.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:23.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=151, tag_tid=11, entry_tid=16495' 2026-02-20T21:11:23.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:23.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=16495 2026-02-20T21:11:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 16495 2026-02-20T21:11:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:23.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:23.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:23.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:23.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:23.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:23.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:23.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:23.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:23.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:23.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:23.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:23.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:23.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:23.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:23.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:23.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:23.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:23.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:23.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:23.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=152, tag_tid=11, entry_tid=16668], [object_number=155, tag_tid=11, entry_tid=16667], [object_number=154, tag_tid=11, entry_tid=16666], [object_number=153, tag_tid=11, entry_tid=16665]]], state=connected] 2026-02-20T21:11:23.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:23.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:23.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:23.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:23.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:23.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:23.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:23.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:23.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:23.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=152, tag_tid=11, entry_tid=16668], [object_number=155, tag_tid=11, entry_tid=16667], [object_number=154, tag_tid=11, entry_tid=16666], [object_number=153, tag_tid=11, entry_tid=16665]]], state=connected] 2026-02-20T21:11:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=152, tag_tid=11, entry_tid=16668' 2026-02-20T21:11:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=152, tag_tid=11, entry_tid=16668' 2026-02-20T21:11:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=11, entry_tid=16668' '!=' 'object_number=151, tag_tid=11, entry_tid=16495' 2026-02-20T21:11:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=11, entry_tid=16668' = 'object_number=151, tag_tid=11, entry_tid=16495' 2026-02-20T21:11:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=152, tag_tid=11, entry_tid=16668' 2026-02-20T21:11:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:23.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=11, entry_tid=16668' 2026-02-20T21:11:23.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:23.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=11, entry_tid=16668' 2026-02-20T21:11:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=16668 2026-02-20T21:11:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 16668 2026-02-20T21:11:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:23.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:24.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:24.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:24.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:24.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:24.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:24.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:24.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:24.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:24.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:24.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:24.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:24.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:24.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=155, tag_tid=11, entry_tid=17223], [object_number=154, tag_tid=11, entry_tid=17222], [object_number=153, tag_tid=11, entry_tid=17221], [object_number=152, tag_tid=11, entry_tid=17220]]], state=connected] 2026-02-20T21:11:24.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:24.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:24.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:24.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:24.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:24.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:24.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:24.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:24.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:24.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:24.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:24.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:24.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:24.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=155, tag_tid=11, entry_tid=17223], [object_number=154, tag_tid=11, entry_tid=17222], [object_number=153, tag_tid=11, entry_tid=17221], [object_number=152, tag_tid=11, entry_tid=17220]]], state=connected] 2026-02-20T21:11:24.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=155, tag_tid=11, entry_tid=17223' 2026-02-20T21:11:24.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=155, tag_tid=11, entry_tid=17223' 2026-02-20T21:11:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=155, tag_tid=11, entry_tid=17223' '!=' 'object_number=152, tag_tid=11, entry_tid=16668' 2026-02-20T21:11:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=155, tag_tid=11, entry_tid=17223' = 'object_number=152, tag_tid=11, entry_tid=16668' 2026-02-20T21:11:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=155, tag_tid=11, entry_tid=17223' 2026-02-20T21:11:24.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:24.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=155, tag_tid=11, entry_tid=17223' 2026-02-20T21:11:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:24.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:24.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:24.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:24.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:24.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:24.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=155, tag_tid=11, entry_tid=17223' 2026-02-20T21:11:24.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:24.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:24.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=17223 2026-02-20T21:11:24.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 17223 2026-02-20T21:11:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:24.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:24.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:24.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:24.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:24.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:24.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:24.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:24.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:24.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:24.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:24.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:24.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:24.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:24.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:24.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:24.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:24.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:24.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:24.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:24.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:24.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:24.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:24.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:24.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:24.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:24.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:24.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:24.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:24.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:24.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:24.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:24.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:24.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:24.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:24.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:24.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=154, tag_tid=11, entry_tid=17570], [object_number=153, tag_tid=11, entry_tid=17569], [object_number=152, tag_tid=11, entry_tid=17568], [object_number=155, tag_tid=11, entry_tid=17567]]], state=connected] 2026-02-20T21:11:24.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:24.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:24.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:24.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:24.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:24.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:24.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=154, tag_tid=11, entry_tid=17570], [object_number=153, tag_tid=11, entry_tid=17569], [object_number=152, tag_tid=11, entry_tid=17568], [object_number=155, tag_tid=11, entry_tid=17567]]], state=connected] 2026-02-20T21:11:24.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=154, tag_tid=11, entry_tid=17570' 2026-02-20T21:11:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=154, tag_tid=11, entry_tid=17570' 2026-02-20T21:11:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=11, entry_tid=17570' '!=' 'object_number=155, tag_tid=11, entry_tid=17223' 2026-02-20T21:11:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=11, entry_tid=17570' = 'object_number=155, tag_tid=11, entry_tid=17223' 2026-02-20T21:11:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=154, tag_tid=11, entry_tid=17570' 2026-02-20T21:11:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=154, tag_tid=11, entry_tid=17570' 2026-02-20T21:11:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:24.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:24.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:24.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:24.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=154, tag_tid=11, entry_tid=17570' 2026-02-20T21:11:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:24.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=17570 2026-02-20T21:11:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 17570 2026-02-20T21:11:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:24.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:24.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:24.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:24.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:24.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:24.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:24.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:24.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:24.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:24.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:24.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:24.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:24.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:24.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:24.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:24.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:24.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:24.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:24.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:24.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:24.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:24.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:24.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:24.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:24.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:24.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=153, tag_tid=11, entry_tid=17781], [object_number=152, tag_tid=11, entry_tid=17780], [object_number=155, tag_tid=11, entry_tid=17779], [object_number=154, tag_tid=11, entry_tid=17778]]], state=connected] 2026-02-20T21:11:24.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:24.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:24.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:24.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:24.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:24.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:24.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:24.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:24.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:24.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:24.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:24.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:24.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:24.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:24.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:24.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:24.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=153, tag_tid=11, entry_tid=17781], [object_number=152, tag_tid=11, entry_tid=17780], [object_number=155, tag_tid=11, entry_tid=17779], [object_number=154, tag_tid=11, entry_tid=17778]]], state=connected] 2026-02-20T21:11:24.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=153, tag_tid=11, entry_tid=17781' 2026-02-20T21:11:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=153, tag_tid=11, entry_tid=17781' 2026-02-20T21:11:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=153, tag_tid=11, entry_tid=17781' '!=' 'object_number=154, tag_tid=11, entry_tid=17570' 2026-02-20T21:11:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=153, tag_tid=11, entry_tid=17781' = 'object_number=154, tag_tid=11, entry_tid=17570' 2026-02-20T21:11:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=153, tag_tid=11, entry_tid=17781' 2026-02-20T21:11:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:24.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:24.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:24.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=11, entry_tid=17781' 2026-02-20T21:11:24.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:24.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:24.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:24.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:24.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:24.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:24.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=11, entry_tid=17781' 2026-02-20T21:11:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:24.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=17781 2026-02-20T21:11:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 17781 2026-02-20T21:11:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:25.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:25.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:25.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:25.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:25.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=154, tag_tid=11, entry_tid=18106], [object_number=153, tag_tid=11, entry_tid=18105], [object_number=152, tag_tid=11, entry_tid=18104], [object_number=155, tag_tid=11, entry_tid=18103]]], state=connected] 2026-02-20T21:11:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:25.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:25.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:25.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:25.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:25.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:25.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=154, tag_tid=11, entry_tid=18106], [object_number=153, tag_tid=11, entry_tid=18105], [object_number=152, tag_tid=11, entry_tid=18104], [object_number=155, tag_tid=11, entry_tid=18103]]], state=connected] 2026-02-20T21:11:25.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=154, tag_tid=11, entry_tid=18106' 2026-02-20T21:11:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=154, tag_tid=11, entry_tid=18106' 2026-02-20T21:11:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=11, entry_tid=18106' '!=' 'object_number=153, tag_tid=11, entry_tid=17781' 2026-02-20T21:11:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=11, entry_tid=18106' = 'object_number=153, tag_tid=11, entry_tid=17781' 2026-02-20T21:11:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=154, tag_tid=11, entry_tid=18106' 2026-02-20T21:11:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:25.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:25.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=154, tag_tid=11, entry_tid=18106' 2026-02-20T21:11:25.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:25.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:25.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:25.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:25.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:25.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=154, tag_tid=11, entry_tid=18106' 2026-02-20T21:11:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:25.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=18106 2026-02-20T21:11:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 18106 2026-02-20T21:11:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:25.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:25.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:25.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:25.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:25.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:25.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=154, tag_tid=11, entry_tid=18426], [object_number=153, tag_tid=11, entry_tid=18425], [object_number=152, tag_tid=11, entry_tid=18424], [object_number=155, tag_tid=11, entry_tid=18423]]], state=connected] 2026-02-20T21:11:25.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:25.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:25.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:25.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:25.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:25.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:25.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:25.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:25.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:25.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=154, tag_tid=11, entry_tid=18426], [object_number=153, tag_tid=11, entry_tid=18425], [object_number=152, tag_tid=11, entry_tid=18424], [object_number=155, tag_tid=11, entry_tid=18423]]], state=connected] 2026-02-20T21:11:25.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=154, tag_tid=11, entry_tid=18426' 2026-02-20T21:11:25.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=154, tag_tid=11, entry_tid=18426' 2026-02-20T21:11:25.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=11, entry_tid=18426' '!=' 'object_number=154, tag_tid=11, entry_tid=18106' 2026-02-20T21:11:25.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:25.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=11, entry_tid=18426' = 'object_number=154, tag_tid=11, entry_tid=18106' 2026-02-20T21:11:25.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=154, tag_tid=11, entry_tid=18426' 2026-02-20T21:11:25.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:25.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:25.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:25.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:25.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=154, tag_tid=11, entry_tid=18426' 2026-02-20T21:11:25.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:25.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:25.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:25.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:25.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:25.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:25.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:25.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=154, tag_tid=11, entry_tid=18426' 2026-02-20T21:11:25.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:25.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:25.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=18426 2026-02-20T21:11:25.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 18426 2026-02-20T21:11:25.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:25.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:25.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:25.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:25.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:25.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:25.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:25.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:25.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:25.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:25.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:25.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:25.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:25.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:25.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:25.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:25.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:25.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:25.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=153, tag_tid=11, entry_tid=18757], [object_number=152, tag_tid=11, entry_tid=18756], [object_number=155, tag_tid=11, entry_tid=18755], [object_number=154, tag_tid=11, entry_tid=18754]]], state=connected] 2026-02-20T21:11:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:25.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:25.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:25.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:25.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:25.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:25.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:25.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=153, tag_tid=11, entry_tid=18757], [object_number=152, tag_tid=11, entry_tid=18756], [object_number=155, tag_tid=11, entry_tid=18755], [object_number=154, tag_tid=11, entry_tid=18754]]], state=connected] 2026-02-20T21:11:25.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=153, tag_tid=11, entry_tid=18757' 2026-02-20T21:11:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=153, tag_tid=11, entry_tid=18757' 2026-02-20T21:11:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=153, tag_tid=11, entry_tid=18757' '!=' 'object_number=154, tag_tid=11, entry_tid=18426' 2026-02-20T21:11:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=153, tag_tid=11, entry_tid=18757' = 'object_number=154, tag_tid=11, entry_tid=18426' 2026-02-20T21:11:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=153, tag_tid=11, entry_tid=18757' 2026-02-20T21:11:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:25.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:25.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:25.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:25.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=11, entry_tid=18757' 2026-02-20T21:11:25.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:25.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:25.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=153, tag_tid=11, entry_tid=18757' 2026-02-20T21:11:25.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:25.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:25.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=18757 2026-02-20T21:11:25.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 18757 2026-02-20T21:11:25.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:25.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:25.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:26.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:26.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:26.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:26.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:26.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:26.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:26.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:26.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:26.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:26.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:26.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:26.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:26.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:26.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:26.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:26.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=155, tag_tid=11, entry_tid=19087], [object_number=154, tag_tid=11, entry_tid=19086], [object_number=153, tag_tid=11, entry_tid=19085], [object_number=152, tag_tid=11, entry_tid=19084]]], state=connected] 2026-02-20T21:11:26.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:26.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:26.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:26.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:26.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:26.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:26.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=155, tag_tid=11, entry_tid=19087], [object_number=154, tag_tid=11, entry_tid=19086], [object_number=153, tag_tid=11, entry_tid=19085], [object_number=152, tag_tid=11, entry_tid=19084]]], state=connected] 2026-02-20T21:11:26.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=155, tag_tid=11, entry_tid=19087' 2026-02-20T21:11:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=155, tag_tid=11, entry_tid=19087' 2026-02-20T21:11:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=155, tag_tid=11, entry_tid=19087' '!=' 'object_number=153, tag_tid=11, entry_tid=18757' 2026-02-20T21:11:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=155, tag_tid=11, entry_tid=19087' = 'object_number=153, tag_tid=11, entry_tid=18757' 2026-02-20T21:11:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=155, tag_tid=11, entry_tid=19087' 2026-02-20T21:11:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=155, tag_tid=11, entry_tid=19087' 2026-02-20T21:11:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=155, tag_tid=11, entry_tid=19087' 2026-02-20T21:11:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=19087 2026-02-20T21:11:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 19087 2026-02-20T21:11:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:26.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:26.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:26.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:26.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:26.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:26.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=152, tag_tid=11, entry_tid=19408], [object_number=155, tag_tid=11, entry_tid=19407], [object_number=154, tag_tid=11, entry_tid=19406], [object_number=153, tag_tid=11, entry_tid=19405]]], state=connected] 2026-02-20T21:11:26.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=152, tag_tid=11, entry_tid=19408], [object_number=155, tag_tid=11, entry_tid=19407], [object_number=154, tag_tid=11, entry_tid=19406], [object_number=153, tag_tid=11, entry_tid=19405]]], state=connected] 2026-02-20T21:11:26.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=152, tag_tid=11, entry_tid=19408' 2026-02-20T21:11:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=152, tag_tid=11, entry_tid=19408' 2026-02-20T21:11:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=11, entry_tid=19408' '!=' 'object_number=155, tag_tid=11, entry_tid=19087' 2026-02-20T21:11:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=11, entry_tid=19408' = 'object_number=155, tag_tid=11, entry_tid=19087' 2026-02-20T21:11:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=152, tag_tid=11, entry_tid=19408' 2026-02-20T21:11:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=11, entry_tid=19408' 2026-02-20T21:11:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:26.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:26.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:26.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:26.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:26.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=11, entry_tid=19408' 2026-02-20T21:11:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=19408 2026-02-20T21:11:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 19408 2026-02-20T21:11:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:26.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:26.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:26.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:26.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:26.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:26.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:26.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:26.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=152, tag_tid=11, entry_tid=19736], [object_number=155, tag_tid=11, entry_tid=19735], [object_number=154, tag_tid=11, entry_tid=19734], [object_number=153, tag_tid=11, entry_tid=19733]]], state=connected] 2026-02-20T21:11:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:26.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:26.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:26.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=152, tag_tid=11, entry_tid=19736], [object_number=155, tag_tid=11, entry_tid=19735], [object_number=154, tag_tid=11, entry_tid=19734], [object_number=153, tag_tid=11, entry_tid=19733]]], state=connected] 2026-02-20T21:11:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=152, tag_tid=11, entry_tid=19736' 2026-02-20T21:11:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=152, tag_tid=11, entry_tid=19736' 2026-02-20T21:11:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=11, entry_tid=19736' '!=' 'object_number=152, tag_tid=11, entry_tid=19408' 2026-02-20T21:11:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=11, entry_tid=19736' = 'object_number=152, tag_tid=11, entry_tid=19408' 2026-02-20T21:11:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=152, tag_tid=11, entry_tid=19736' 2026-02-20T21:11:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=11, entry_tid=19736' 2026-02-20T21:11:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:26.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:26.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:26.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:26.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:26.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=11, entry_tid=19736' 2026-02-20T21:11:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=19736 2026-02-20T21:11:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 19736 2026-02-20T21:11:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:27.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:27.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:27.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:27.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:27.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:27.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:27.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:27.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:27.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=152, tag_tid=11, entry_tid=20040], [object_number=155, tag_tid=11, entry_tid=20039], [object_number=154, tag_tid=11, entry_tid=20038], [object_number=153, tag_tid=11, entry_tid=20037]]], state=connected] 2026-02-20T21:11:27.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=152, tag_tid=11, entry_tid=20040], [object_number=155, tag_tid=11, entry_tid=20039], [object_number=154, tag_tid=11, entry_tid=20038], [object_number=153, tag_tid=11, entry_tid=20037]]], state=connected] 2026-02-20T21:11:27.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=152, tag_tid=11, entry_tid=20040' 2026-02-20T21:11:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=152, tag_tid=11, entry_tid=20040' 2026-02-20T21:11:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=11, entry_tid=20040' '!=' 'object_number=152, tag_tid=11, entry_tid=19736' 2026-02-20T21:11:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=11, entry_tid=20040' = 'object_number=152, tag_tid=11, entry_tid=19736' 2026-02-20T21:11:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=152, tag_tid=11, entry_tid=20040' 2026-02-20T21:11:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=11, entry_tid=20040' 2026-02-20T21:11:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:27.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=11, entry_tid=20040' 2026-02-20T21:11:27.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:27.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:27.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=20040 2026-02-20T21:11:27.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 20040 2026-02-20T21:11:27.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:27.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:27.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:27.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:27.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:27.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:27.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=152, tag_tid=11, entry_tid=20332], [object_number=155, tag_tid=11, entry_tid=20331], [object_number=154, tag_tid=11, entry_tid=20330], [object_number=153, tag_tid=11, entry_tid=20329]]], state=connected] 2026-02-20T21:11:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:27.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:27.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:27.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:27.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:27.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:27.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=152, tag_tid=11, entry_tid=20332], [object_number=155, tag_tid=11, entry_tid=20331], [object_number=154, tag_tid=11, entry_tid=20330], [object_number=153, tag_tid=11, entry_tid=20329]]], state=connected] 2026-02-20T21:11:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=152, tag_tid=11, entry_tid=20332' 2026-02-20T21:11:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=152, tag_tid=11, entry_tid=20332' 2026-02-20T21:11:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=11, entry_tid=20332' '!=' 'object_number=152, tag_tid=11, entry_tid=20040' 2026-02-20T21:11:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=152, tag_tid=11, entry_tid=20332' = 'object_number=152, tag_tid=11, entry_tid=20040' 2026-02-20T21:11:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=152, tag_tid=11, entry_tid=20332' 2026-02-20T21:11:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=11, entry_tid=20332' 2026-02-20T21:11:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=152, tag_tid=11, entry_tid=20332' 2026-02-20T21:11:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=20332 2026-02-20T21:11:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 20332 2026-02-20T21:11:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:27.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:27.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:27.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:27.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:27.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:27.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:27.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:27.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:27.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:27.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:27.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:27.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:27.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:27.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:27.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=154, tag_tid=11, entry_tid=20634], [object_number=153, tag_tid=11, entry_tid=20633], [object_number=152, tag_tid=11, entry_tid=20632], [object_number=155, tag_tid=11, entry_tid=20631]]], state=connected] 2026-02-20T21:11:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:27.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:27.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:27.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:27.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:27.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:27.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 38 2026-02-20T21:11:27.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:27.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:27.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:27.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=154, tag_tid=11, entry_tid=20634], [object_number=153, tag_tid=11, entry_tid=20633], [object_number=152, tag_tid=11, entry_tid=20632], [object_number=155, tag_tid=11, entry_tid=20631]]], state=connected] 2026-02-20T21:11:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=154, tag_tid=11, entry_tid=20634' 2026-02-20T21:11:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=154, tag_tid=11, entry_tid=20634' 2026-02-20T21:11:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=11, entry_tid=20634' '!=' 'object_number=152, tag_tid=11, entry_tid=20332' 2026-02-20T21:11:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=154, tag_tid=11, entry_tid=20634' = 'object_number=152, tag_tid=11, entry_tid=20332' 2026-02-20T21:11:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=154, tag_tid=11, entry_tid=20634' 2026-02-20T21:11:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=154, tag_tid=11, entry_tid=20634' 2026-02-20T21:11:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=154, tag_tid=11, entry_tid=20634' 2026-02-20T21:11:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=20634 2026-02-20T21:11:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 20634 2026-02-20T21:11:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:28.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:28.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=20859], [object_number=158, tag_tid=11, entry_tid=20858], [object_number=157, tag_tid=11, entry_tid=20857], [object_number=156, tag_tid=11, entry_tid=20856]]], state=connected] 2026-02-20T21:11:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=20859], [object_number=158, tag_tid=11, entry_tid=20858], [object_number=157, tag_tid=11, entry_tid=20857], [object_number=156, tag_tid=11, entry_tid=20856]]], state=connected] 2026-02-20T21:11:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=159, tag_tid=11, entry_tid=20859' 2026-02-20T21:11:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=159, tag_tid=11, entry_tid=20859' 2026-02-20T21:11:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=20859' '!=' 'object_number=154, tag_tid=11, entry_tid=20634' 2026-02-20T21:11:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=20859' = 'object_number=154, tag_tid=11, entry_tid=20634' 2026-02-20T21:11:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=159, tag_tid=11, entry_tid=20859' 2026-02-20T21:11:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=20859' 2026-02-20T21:11:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:28.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=20859' 2026-02-20T21:11:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=20859 2026-02-20T21:11:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 20859 2026-02-20T21:11:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:28.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:28.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:28.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:28.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:28.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:28.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:28.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:28.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:28.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:28.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:28.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:28.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:28.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:28.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:28.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:28.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:28.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:28.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:28.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:28.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:28.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:28.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:28.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:28.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:28.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=21211], [object_number=158, tag_tid=11, entry_tid=21210], [object_number=157, tag_tid=11, entry_tid=21209], [object_number=156, tag_tid=11, entry_tid=21208]]], state=connected] 2026-02-20T21:11:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:28.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:28.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:28.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:28.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:28.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:28.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=21211], [object_number=158, tag_tid=11, entry_tid=21210], [object_number=157, tag_tid=11, entry_tid=21209], [object_number=156, tag_tid=11, entry_tid=21208]]], state=connected] 2026-02-20T21:11:28.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=159, tag_tid=11, entry_tid=21211' 2026-02-20T21:11:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=159, tag_tid=11, entry_tid=21211' 2026-02-20T21:11:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=21211' '!=' 'object_number=159, tag_tid=11, entry_tid=20859' 2026-02-20T21:11:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=21211' = 'object_number=159, tag_tid=11, entry_tid=20859' 2026-02-20T21:11:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=159, tag_tid=11, entry_tid=21211' 2026-02-20T21:11:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:28.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=21211' 2026-02-20T21:11:28.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:28.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:28.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:28.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:28.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=21211' 2026-02-20T21:11:28.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:28.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:28.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=21211 2026-02-20T21:11:28.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 21211 2026-02-20T21:11:28.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:28.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:28.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:28.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:28.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:28.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:28.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:28.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:28.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:28.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=158, tag_tid=11, entry_tid=21550], [object_number=157, tag_tid=11, entry_tid=21549], [object_number=156, tag_tid=11, entry_tid=21548], [object_number=159, tag_tid=11, entry_tid=21547]]], state=connected] 2026-02-20T21:11:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:28.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=158, tag_tid=11, entry_tid=21550], [object_number=157, tag_tid=11, entry_tid=21549], [object_number=156, tag_tid=11, entry_tid=21548], [object_number=159, tag_tid=11, entry_tid=21547]]], state=connected] 2026-02-20T21:11:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=158, tag_tid=11, entry_tid=21550' 2026-02-20T21:11:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=158, tag_tid=11, entry_tid=21550' 2026-02-20T21:11:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=158, tag_tid=11, entry_tid=21550' '!=' 'object_number=159, tag_tid=11, entry_tid=21211' 2026-02-20T21:11:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=158, tag_tid=11, entry_tid=21550' = 'object_number=159, tag_tid=11, entry_tid=21211' 2026-02-20T21:11:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=158, tag_tid=11, entry_tid=21550' 2026-02-20T21:11:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=158, tag_tid=11, entry_tid=21550' 2026-02-20T21:11:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:29.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:29.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=158, tag_tid=11, entry_tid=21550' 2026-02-20T21:11:29.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:29.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=21550 2026-02-20T21:11:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 21550 2026-02-20T21:11:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:29.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:29.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:29.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:29.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:29.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:29.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:29.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:29.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:29.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:29.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:29.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:29.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:29.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:29.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:29.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=157, tag_tid=11, entry_tid=21933], [object_number=156, tag_tid=11, entry_tid=21932], [object_number=159, tag_tid=11, entry_tid=21931], [object_number=158, tag_tid=11, entry_tid=21930]]], state=connected] 2026-02-20T21:11:29.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:29.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=157, tag_tid=11, entry_tid=21933], [object_number=156, tag_tid=11, entry_tid=21932], [object_number=159, tag_tid=11, entry_tid=21931], [object_number=158, tag_tid=11, entry_tid=21930]]], state=connected] 2026-02-20T21:11:29.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=157, tag_tid=11, entry_tid=21933' 2026-02-20T21:11:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=157, tag_tid=11, entry_tid=21933' 2026-02-20T21:11:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=157, tag_tid=11, entry_tid=21933' '!=' 'object_number=158, tag_tid=11, entry_tid=21550' 2026-02-20T21:11:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=157, tag_tid=11, entry_tid=21933' = 'object_number=158, tag_tid=11, entry_tid=21550' 2026-02-20T21:11:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=157, tag_tid=11, entry_tid=21933' 2026-02-20T21:11:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:29.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:29.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=11, entry_tid=21933' 2026-02-20T21:11:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:29.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:29.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:29.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:29.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:29.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:29.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=11, entry_tid=21933' 2026-02-20T21:11:29.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:29.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:29.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=21933 2026-02-20T21:11:29.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 21933 2026-02-20T21:11:29.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:29.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:29.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:29.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:29.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:29.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:29.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=157, tag_tid=11, entry_tid=22273], [object_number=156, tag_tid=11, entry_tid=22272], [object_number=159, tag_tid=11, entry_tid=22271], [object_number=158, tag_tid=11, entry_tid=22270]]], state=connected] 2026-02-20T21:11:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:29.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:29.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:29.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:29.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:29.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:29.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:29.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:29.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:29.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:29.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:29.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:29.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:29.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:29.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:29.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:29.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=157, tag_tid=11, entry_tid=22273], [object_number=156, tag_tid=11, entry_tid=22272], [object_number=159, tag_tid=11, entry_tid=22271], [object_number=158, tag_tid=11, entry_tid=22270]]], state=connected] 2026-02-20T21:11:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=157, tag_tid=11, entry_tid=22273' 2026-02-20T21:11:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=157, tag_tid=11, entry_tid=22273' 2026-02-20T21:11:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=157, tag_tid=11, entry_tid=22273' '!=' 'object_number=157, tag_tid=11, entry_tid=21933' 2026-02-20T21:11:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=157, tag_tid=11, entry_tid=22273' = 'object_number=157, tag_tid=11, entry_tid=21933' 2026-02-20T21:11:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=157, tag_tid=11, entry_tid=22273' 2026-02-20T21:11:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:29.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:29.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:29.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=11, entry_tid=22273' 2026-02-20T21:11:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:29.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:29.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:29.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=11, entry_tid=22273' 2026-02-20T21:11:29.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:29.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=22273 2026-02-20T21:11:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 22273 2026-02-20T21:11:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:29.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:29.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:29.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:29.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:29.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:29.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:29.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:29.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:29.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:29.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:29.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:29.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:29.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:29.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:29.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:29.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:29.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:29.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:29.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:29.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=156, tag_tid=11, entry_tid=22616], [object_number=159, tag_tid=11, entry_tid=22615], [object_number=158, tag_tid=11, entry_tid=22614], [object_number=157, tag_tid=11, entry_tid=22613]]], state=connected] 2026-02-20T21:11:29.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:29.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:29.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:29.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=156, tag_tid=11, entry_tid=22616], [object_number=159, tag_tid=11, entry_tid=22615], [object_number=158, tag_tid=11, entry_tid=22614], [object_number=157, tag_tid=11, entry_tid=22613]]], state=connected] 2026-02-20T21:11:30.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=156, tag_tid=11, entry_tid=22616' 2026-02-20T21:11:30.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=156, tag_tid=11, entry_tid=22616' 2026-02-20T21:11:30.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=156, tag_tid=11, entry_tid=22616' '!=' 'object_number=157, tag_tid=11, entry_tid=22273' 2026-02-20T21:11:30.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:30.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=156, tag_tid=11, entry_tid=22616' = 'object_number=157, tag_tid=11, entry_tid=22273' 2026-02-20T21:11:30.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=156, tag_tid=11, entry_tid=22616' 2026-02-20T21:11:30.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:30.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:30.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:30.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:30.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=156, tag_tid=11, entry_tid=22616' 2026-02-20T21:11:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:30.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:30.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=156, tag_tid=11, entry_tid=22616' 2026-02-20T21:11:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=22616 2026-02-20T21:11:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 22616 2026-02-20T21:11:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:30.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:30.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:30.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:30.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:30.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:30.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:30.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=22903], [object_number=158, tag_tid=11, entry_tid=22902], [object_number=157, tag_tid=11, entry_tid=22901], [object_number=156, tag_tid=11, entry_tid=22900]]], state=connected] 2026-02-20T21:11:30.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:30.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=22903], [object_number=158, tag_tid=11, entry_tid=22902], [object_number=157, tag_tid=11, entry_tid=22901], [object_number=156, tag_tid=11, entry_tid=22900]]], state=connected] 2026-02-20T21:11:30.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=159, tag_tid=11, entry_tid=22903' 2026-02-20T21:11:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=159, tag_tid=11, entry_tid=22903' 2026-02-20T21:11:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=22903' '!=' 'object_number=156, tag_tid=11, entry_tid=22616' 2026-02-20T21:11:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=22903' = 'object_number=156, tag_tid=11, entry_tid=22616' 2026-02-20T21:11:30.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=159, tag_tid=11, entry_tid=22903' 2026-02-20T21:11:30.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:30.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:30.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=22903' 2026-02-20T21:11:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:30.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:30.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=22903' 2026-02-20T21:11:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=22903 2026-02-20T21:11:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 22903 2026-02-20T21:11:30.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:30.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:30.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:30.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:30.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:30.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:30.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:30.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:30.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:30.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:30.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:30.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:30.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:30.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:30.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:30.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:30.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:30.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:30.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:30.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:30.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:30.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:30.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:30.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=157, tag_tid=11, entry_tid=23221], [object_number=156, tag_tid=11, entry_tid=23220], [object_number=159, tag_tid=11, entry_tid=23219], [object_number=158, tag_tid=11, entry_tid=23218]]], state=connected] 2026-02-20T21:11:30.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:30.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:30.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:30.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:30.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:30.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:30.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=157, tag_tid=11, entry_tid=23221], [object_number=156, tag_tid=11, entry_tid=23220], [object_number=159, tag_tid=11, entry_tid=23219], [object_number=158, tag_tid=11, entry_tid=23218]]], state=connected] 2026-02-20T21:11:30.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=157, tag_tid=11, entry_tid=23221' 2026-02-20T21:11:30.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=157, tag_tid=11, entry_tid=23221' 2026-02-20T21:11:30.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=157, tag_tid=11, entry_tid=23221' '!=' 'object_number=159, tag_tid=11, entry_tid=22903' 2026-02-20T21:11:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=157, tag_tid=11, entry_tid=23221' = 'object_number=159, tag_tid=11, entry_tid=22903' 2026-02-20T21:11:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=157, tag_tid=11, entry_tid=23221' 2026-02-20T21:11:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:30.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=11, entry_tid=23221' 2026-02-20T21:11:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:30.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:30.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:30.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:30.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:30.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:30.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=11, entry_tid=23221' 2026-02-20T21:11:30.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:30.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=23221 2026-02-20T21:11:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 23221 2026-02-20T21:11:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:31.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:31.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:31.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:31.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=158, tag_tid=11, entry_tid=23542], [object_number=157, tag_tid=11, entry_tid=23541], [object_number=156, tag_tid=11, entry_tid=23540], [object_number=159, tag_tid=11, entry_tid=23539]]], state=connected] 2026-02-20T21:11:31.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:31.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:31.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:31.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:31.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:31.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:31.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:31.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:31.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:31.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:31.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:31.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:31.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=158, tag_tid=11, entry_tid=23542], [object_number=157, tag_tid=11, entry_tid=23541], [object_number=156, tag_tid=11, entry_tid=23540], [object_number=159, tag_tid=11, entry_tid=23539]]], state=connected] 2026-02-20T21:11:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=158, tag_tid=11, entry_tid=23542' 2026-02-20T21:11:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=158, tag_tid=11, entry_tid=23542' 2026-02-20T21:11:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=158, tag_tid=11, entry_tid=23542' '!=' 'object_number=157, tag_tid=11, entry_tid=23221' 2026-02-20T21:11:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=158, tag_tid=11, entry_tid=23542' = 'object_number=157, tag_tid=11, entry_tid=23221' 2026-02-20T21:11:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=158, tag_tid=11, entry_tid=23542' 2026-02-20T21:11:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=158, tag_tid=11, entry_tid=23542' 2026-02-20T21:11:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:31.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:31.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:31.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:31.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:31.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:31.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=158, tag_tid=11, entry_tid=23542' 2026-02-20T21:11:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=23542 2026-02-20T21:11:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 23542 2026-02-20T21:11:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:31.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:31.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:31.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:31.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:31.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:31.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:31.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:31.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:31.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:31.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:31.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:31.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:31.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:31.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:31.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:31.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:31.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=156, tag_tid=11, entry_tid=23872], [object_number=159, tag_tid=11, entry_tid=23871], [object_number=158, tag_tid=11, entry_tid=23870], [object_number=157, tag_tid=11, entry_tid=23869]]], state=connected] 2026-02-20T21:11:31.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:31.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:31.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:31.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:31.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:31.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:31.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:31.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:31.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:31.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=156, tag_tid=11, entry_tid=23872], [object_number=159, tag_tid=11, entry_tid=23871], [object_number=158, tag_tid=11, entry_tid=23870], [object_number=157, tag_tid=11, entry_tid=23869]]], state=connected] 2026-02-20T21:11:31.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=156, tag_tid=11, entry_tid=23872' 2026-02-20T21:11:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=156, tag_tid=11, entry_tid=23872' 2026-02-20T21:11:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=156, tag_tid=11, entry_tid=23872' '!=' 'object_number=158, tag_tid=11, entry_tid=23542' 2026-02-20T21:11:31.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=156, tag_tid=11, entry_tid=23872' = 'object_number=158, tag_tid=11, entry_tid=23542' 2026-02-20T21:11:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=156, tag_tid=11, entry_tid=23872' 2026-02-20T21:11:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=156, tag_tid=11, entry_tid=23872' 2026-02-20T21:11:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:31.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=156, tag_tid=11, entry_tid=23872' 2026-02-20T21:11:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=23872 2026-02-20T21:11:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 23872 2026-02-20T21:11:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:31.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:31.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:31.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:31.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:31.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:31.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:31.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:31.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:31.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:31.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:31.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:31.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:31.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:31.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:31.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:31.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:31.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=158, tag_tid=11, entry_tid=24210], [object_number=157, tag_tid=11, entry_tid=24209], [object_number=156, tag_tid=11, entry_tid=24208], [object_number=159, tag_tid=11, entry_tid=24207]]], state=connected] 2026-02-20T21:11:31.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:31.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:31.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:31.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:31.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:31.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:31.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:31.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:31.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:31.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:31.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:31.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:31.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:31.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:31.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:31.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=158, tag_tid=11, entry_tid=24210], [object_number=157, tag_tid=11, entry_tid=24209], [object_number=156, tag_tid=11, entry_tid=24208], [object_number=159, tag_tid=11, entry_tid=24207]]], state=connected] 2026-02-20T21:11:31.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=158, tag_tid=11, entry_tid=24210' 2026-02-20T21:11:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=158, tag_tid=11, entry_tid=24210' 2026-02-20T21:11:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=158, tag_tid=11, entry_tid=24210' '!=' 'object_number=156, tag_tid=11, entry_tid=23872' 2026-02-20T21:11:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=158, tag_tid=11, entry_tid=24210' = 'object_number=156, tag_tid=11, entry_tid=23872' 2026-02-20T21:11:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=158, tag_tid=11, entry_tid=24210' 2026-02-20T21:11:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:31.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=158, tag_tid=11, entry_tid=24210' 2026-02-20T21:11:31.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:31.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:31.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:31.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:31.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=158, tag_tid=11, entry_tid=24210' 2026-02-20T21:11:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:31.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24210 2026-02-20T21:11:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 24210 2026-02-20T21:11:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:31.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:31.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:31.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:31.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:31.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:31.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:31.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:31.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:31.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:31.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:31.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:32.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:32.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:32.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:32.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=24595], [object_number=158, tag_tid=11, entry_tid=24594], [object_number=157, tag_tid=11, entry_tid=24593], [object_number=156, tag_tid=11, entry_tid=24592]]], state=connected] 2026-02-20T21:11:32.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:32.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:32.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:32.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:32.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:32.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:32.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:32.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=24595], [object_number=158, tag_tid=11, entry_tid=24594], [object_number=157, tag_tid=11, entry_tid=24593], [object_number=156, tag_tid=11, entry_tid=24592]]], state=connected] 2026-02-20T21:11:32.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=159, tag_tid=11, entry_tid=24595' 2026-02-20T21:11:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=159, tag_tid=11, entry_tid=24595' 2026-02-20T21:11:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=24595' '!=' 'object_number=158, tag_tid=11, entry_tid=24210' 2026-02-20T21:11:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=24595' = 'object_number=158, tag_tid=11, entry_tid=24210' 2026-02-20T21:11:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=159, tag_tid=11, entry_tid=24595' 2026-02-20T21:11:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:32.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:32.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=24595' 2026-02-20T21:11:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:32.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:32.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=24595' 2026-02-20T21:11:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24595 2026-02-20T21:11:32.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 24595 2026-02-20T21:11:32.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:32.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:32.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:32.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:32.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:32.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:32.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:32.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:32.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:32.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:32.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:32.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:32.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:32.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:32.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:32.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:32.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:32.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:32.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:32.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838], [object_number=157, tag_tid=11, entry_tid=24837], [object_number=156, tag_tid=11, entry_tid=24836]]], state=connected] 2026-02-20T21:11:32.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:32.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:32.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:32.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:32.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:32.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838], [object_number=157, tag_tid=11, entry_tid=24837], [object_number=156, tag_tid=11, entry_tid=24836]]], state=connected] 2026-02-20T21:11:32.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=159, tag_tid=11, entry_tid=24839' 2026-02-20T21:11:32.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=159, tag_tid=11, entry_tid=24839' 2026-02-20T21:11:32.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=24839' '!=' 'object_number=159, tag_tid=11, entry_tid=24595' 2026-02-20T21:11:32.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:11:32.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=24839' = 'object_number=159, tag_tid=11, entry_tid=24595' 2026-02-20T21:11:32.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=159, tag_tid=11, entry_tid=24839' 2026-02-20T21:11:32.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:32.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:32.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:32.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=12 2026-02-20T21:11:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=24839' 2026-02-20T21:11:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:11:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=11 2026-02-20T21:11:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:32.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:11:32.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=159, tag_tid=11, entry_tid=24839' 2026-02-20T21:11:32.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:11:32.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:11:32.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24839 2026-02-20T21:11:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 12 = 11 -a 1 -le 24839 2026-02-20T21:11:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:11:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:32.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:32.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:32.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:32.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:32.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:32.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:32.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:32.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:32.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:32.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:32.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:32.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:32.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:32.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:32.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:32.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:32.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:32.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:32.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:32.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:32.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:32.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838], [object_number=157, tag_tid=11, entry_tid=24837], [object_number=156, tag_tid=11, entry_tid=24836]]], state=connected] 2026-02-20T21:11:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:32.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838], [object_number=157, tag_tid=11, entry_tid=24837], [object_number=156, tag_tid=11, entry_tid=24836]]], state=connected] 2026-02-20T21:11:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:32.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=159, tag_tid=11, entry_tid=24839' 2026-02-20T21:11:32.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=159, tag_tid=11, entry_tid=24839' 2026-02-20T21:11:32.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=159, tag_tid=11, entry_tid=24839' '!=' 'object_number=159, tag_tid=11, entry_tid=24839' 2026-02-20T21:11:32.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:11:32.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:11:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:11:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:11:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:11:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:11:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:11:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:11:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:11:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:11:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:11:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:11:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:11:33.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:11:33.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:11:33.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:11:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:11:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:11:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:11:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:11:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:11:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:11:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:11:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:11:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:11:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:33.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:11:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:11:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:11:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:11:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:11:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:11:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:11:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:11:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:11:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:11:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:11:33.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:11:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:11:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:11:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:11:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:11:33.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:11:33.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:33.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:11:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:11:33.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:33.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:11:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:11:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:11:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:33.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:33.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:11:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:33.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:11:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:11:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:11:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:11:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:11:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:11:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:11:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:11:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:11:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:33.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 39 2026-02-20T21:11:33.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 39 2026-02-20T21:11:33.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:11:33.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:33.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=157, tag_tid=12, entry_tid=1], [object_number=156, tag_tid=12, entry_tid=0], [object_number=159, tag_tid=11, entry_tid=24839], [object_number=158, tag_tid=11, entry_tid=24838]]], state=connected] 2026-02-20T21:11:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:11:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=157, tag_tid=12, entry_tid=1' -a 'object_number=157, tag_tid=12, entry_tid=1' = 'object_number=157, tag_tid=12, entry_tid=1' 2026-02-20T21:11:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap4 2026-02-20T21:11:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:33.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4 2026-02-20T21:11:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:11:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:11:33.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:11:34.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap4 2026-02-20T21:11:34.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-20T21:11:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:11:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:11:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 7 2026-02-20T21:11:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ba958b175d 2026-02-20T21:11:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ba958b175d 2026-02-20T21:11:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:11:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:11:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:11:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:11:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:06:40 2026 2026-02-20T21:11:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:10:32 2026 2026-02-20T21:11:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:11:28 2026 2026-02-20T21:11:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:11:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ba958b175d 2026-02-20T21:11:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:11:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:11:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:11:34.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:11:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap3-clean ']' 2026-02-20T21:11:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap3-clean 2026-02-20T21:11:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3-clean 2026-02-20T21:11:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:11:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:11:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:11:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:11:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:11:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:11:34.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:11:34.713 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.711+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:11:34.715 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.713+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:34.715 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.713+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:34.715 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.713+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:34.716 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.713+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:34.869 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.867+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:34.869 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.867+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:34.870 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.868+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:34.871 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.869+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:34.871 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.869+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:34.871 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.869+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:34.906 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.904+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:34.907 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.904+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:34.908 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.905+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:34.908 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.905+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:34.908 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:34.905+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:35.013 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:35.011+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:11:35.014 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:35.012+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:35.014 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:35.012+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:35.014 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:35.012+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:35.015 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:35.014+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:35.085 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:11:35.083+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:11:35.085 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:11:35.083+0000 7f412fcd2640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:11:35.086 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:11:35.084+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:11:35.086 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:11:35.084+0000 7f412a4c7640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:11:35.086 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:11:35.084+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:11:35.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:11:35.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:11:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:11:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap4 2026-02-20T21:11:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4 2026-02-20T21:11:35.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:11:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:11:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:11:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:11:35.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:11:35.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:11:35.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:11:35.873 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:35.871+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:35.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:35.873+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:11:35.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:35.873+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:11:35.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:35.874+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:11:35.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:35.874+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:11:35.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:35.874+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:11:36.027 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-20T21:11:36.027 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-20T21:11:36.027 INFO:tasks.rbd_mirror_thrash:waiting for 8.681127746199271 secs before reviving daemons 2026-02-20T21:11:36.028 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:11:36.027+0000 7f2deca13640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 47235) UID: 1000 2026-02-20T21:11:36.030 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.028+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:11:36.031 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.029+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:36.032 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.029+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:36.032 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.030+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:36.032 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.030+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:36.032 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.030+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:11:36.069 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.066+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:11:36.069 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.067+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:11:36.140 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.137+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:11:36.140 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.138+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:36.141 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.138+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:36.141 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.138+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:36.141 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.138+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:36.177 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.175+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:11:36.177 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.175+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:11:36.178 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.176+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:36.178 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:11:36.176+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:11:36.246 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:11:36.245+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:11:36.247 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:11:36.245+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:11:36.248 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:11:36.246+0000 7f412a4c7640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:11:36.248 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:11:36.246+0000 7f412fcd2640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:11:36.248 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:11:36.246+0000 7f412ecd0640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:11:36.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:11:36.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:11:36.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:11:36.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:36.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap4-clean 2026-02-20T21:11:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap4-clean 2026-02-20T21:11:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:11:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4-clean 2026-02-20T21:11:36.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap4-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:11:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:11:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:11:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:11:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:11:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-20T21:11:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:11:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:11:36.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 10140672 bytes 2026-02-20T21:11:37.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 22867968 bytes 2026-02-20T21:11:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 87796736 bytes 2026-02-20T21:11:38.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 139476992 bytes 2026-02-20T21:11:39.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 169391104 bytes 2026-02-20T21:11:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 216423424 bytes 2026-02-20T21:11:40.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 288378880 bytes 2026-02-20T21:11:41.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 330576896 bytes 2026-02-20T21:11:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 378378240 bytes 2026-02-20T21:11:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 424251392 bytes 2026-02-20T21:11:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 457219072 bytes 2026-02-20T21:11:43.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 486462464 bytes 2026-02-20T21:11:44.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 501275648 bytes 2026-02-20T21:11:44.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 512430080 bytes 2026-02-20T21:11:44.709 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-20T21:11:44.710 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:11:44.710 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-20T21:11:44.710 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T21:11:44.710 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T21:11:44.713 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T21:11:44.713 INFO:tasks.rbd_mirror_thrash:waiting for 102.71651946191668 secs before thrashing 2026-02-20T21:11:45.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 541793280 bytes 2026-02-20T21:11:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 585512960 bytes 2026-02-20T21:11:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 630409216 bytes 2026-02-20T21:11:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 672389120 bytes 2026-02-20T21:11:47.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 708414464 bytes 2026-02-20T21:11:47.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 734614528 bytes 2026-02-20T21:11:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 767300608 bytes 2026-02-20T21:11:48.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2200 IOs, 774457344 bytes 2026-02-20T21:11:48.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2300 IOs, 829220864 bytes 2026-02-20T21:11:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2400 IOs, 833204224 bytes 2026-02-20T21:11:48.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2500 IOs, 849526784 bytes 2026-02-20T21:11:49.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2600 IOs, 880902144 bytes 2026-02-20T21:11:50.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2700 IOs, 983996416 bytes 2026-02-20T21:11:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:11:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:11:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:11:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:11:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:50.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:11:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:11:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:11:50.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:11:51.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:11:51.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:51.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:51.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:51.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:51.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:11:51.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:11:51.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:11:51.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:11:51.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:11:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:11:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:11:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-20T21:11:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":13304557.07,"entries_behind_primary":0,"entries_per_second":828.13,"non_primary_position":{"entry_tid":1,"object_number":157,"tag_tid":12},"primary_position":{"entry_tid":1,"object_number":157,"tag_tid":12}} 2026-02-20T21:11:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.0 on trial163 2026-02-20T21:11:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:11:38 2026-02-20T21:11:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:11:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:11:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:11:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:11:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:11:38' 2026-02-20T21:11:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:11:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-20T21:11:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-20T21:11:51.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":13304557.07,"entries_behind_primary":0,"entries_per_second":828.13,"non_primary_position":{"entry_tid":1,"object_number":157,"tag_tid":12},"primary_position":{"entry_tid":1,"object_number":157,"tag_tid":12}} 2026-02-20T21:11:51.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:11:51.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:11:51.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-20T21:11:51.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:11:51.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:11:51.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.0 on trial163 2026-02-20T21:11:51.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-20T21:11:51.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:11:51.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:11:38' 2026-02-20T21:11:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:11:38' 2026-02-20T21:11:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:11:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:11:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-20 21:11:38' 2026-02-20T21:11:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:11:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:11:51.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:11:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142044287mirror.0trial16320.2.0-677-gf7870454trueOK43304332mirror.2trial16320.2.0-677-gf7870454falseOK43484351mirror.3trial16320.2.0-677-gf7870454falseOK43884391mirror.1trial16320.2.0-677-gf7870454falseOKtestba923525-490d-4a76-8e5f-b742509a94daup+replayingreplaying, {"bytes_per_second":13304557.07,"entries_behind_primary":0,"entries_per_second":828.13,"non_primary_position":{"entry_tid":1,"object_number":157,"tag_tid":12},"primary_position":{"entry_tid":1,"object_number":157,"tag_tid":12}}42044287mirror.0trial1632026-02-20 21:11:38cluster28b968eaf-6c24-469f-bea5-c8acb780219eup+stoppedlocal image is primary2026-02-20 21:11:38' 2026-02-20T21:11:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-20T21:11:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:11:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:11:38' 2026-02-20T21:11:51.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:11:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-20T21:11:51.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:11:51.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-20T21:11:51.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:11:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:11:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap5 2026-02-20T21:11:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap5 2026-02-20T21:11:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:11:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:11:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:11:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5 2026-02-20T21:11:51.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:12:08.430 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.47269581531660376) > weight (0.25) 2026-02-20T21:12:08.431 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-20T21:12:08.431 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-20T21:12:08.431 INFO:tasks.rbd_mirror_thrash:waiting for 0.3206179628359551 secs before reviving daemons 2026-02-20T21:12:08.432 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:12:08.430+0000 7f626da99640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 60274) UID: 1000 2026-02-20T21:12:08.753 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-20T21:12:08.753 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:12:08.753 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-20T21:12:08.754 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-20T21:12:08.754 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-20T21:12:08.756 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-20T21:12:08.756 INFO:tasks.rbd_mirror_thrash:waiting for 90.22030481629892 secs before thrashing 2026-02-20T21:12:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:12:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:12:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:12:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:12:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:12:22.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:12:22.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:22.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:22.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:22.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:12:22.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:12:22.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:12:22.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:12:22.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:12:22.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:12:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:12:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:12:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:12:22.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:12:22.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:22.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:12:22.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:12:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:12:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:12:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:12:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:22.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:12:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:12:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:12:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:12:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:12:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:12:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:12:22.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:12:22.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:12:22.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:22.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:22.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:12:22.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:12:22.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:12:22.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:12:22.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:22.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:22.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:12:22.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:12:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:12:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:12:22.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:12:22.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:12:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:12:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:12:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:12:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:12:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:12:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:12:22.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:12:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:12:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:12:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:12:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:12:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:12:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:12:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:12:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:12:22.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:22.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:23.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:23.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:23.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:23.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:23.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:23.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:23.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:23.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:23.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:23.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:23.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:23.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:23.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:23.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:23.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:23.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:23.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:23.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:23.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:23.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:23.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:23.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:23.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:23.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:23.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:23.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:23.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 49 2026-02-20T21:12:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:23.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=196, tag_tid=14, entry_tid=40892], [object_number=199, tag_tid=14, entry_tid=40891], [object_number=198, tag_tid=14, entry_tid=40890], [object_number=197, tag_tid=14, entry_tid=40889]]], state=connected] 2026-02-20T21:12:23.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:23.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:23.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:23.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:23.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:23.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 49 2026-02-20T21:12:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:23.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=196, tag_tid=14, entry_tid=40892], [object_number=199, tag_tid=14, entry_tid=40891], [object_number=198, tag_tid=14, entry_tid=40890], [object_number=197, tag_tid=14, entry_tid=40889]]], state=connected] 2026-02-20T21:12:23.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=196, tag_tid=14, entry_tid=40892' 2026-02-20T21:12:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=196, tag_tid=14, entry_tid=40892' 2026-02-20T21:12:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=196, tag_tid=14, entry_tid=40892' '!=' '' 2026-02-20T21:12:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=196, tag_tid=14, entry_tid=40892' = '' 2026-02-20T21:12:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=196, tag_tid=14, entry_tid=40892' 2026-02-20T21:12:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:23.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:23.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:23.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=196, tag_tid=14, entry_tid=40892' 2026-02-20T21:12:23.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:23.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:23.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:23.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:23.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:23.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:23.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=196, tag_tid=14, entry_tid=40892' 2026-02-20T21:12:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:23.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40892 2026-02-20T21:12:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 40892 2026-02-20T21:12:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:23.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:23.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:23.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:23.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:23.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:23.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:23.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:23.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:23.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:23.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:23.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:23.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:23.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:23.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:23.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:23.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:23.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:23.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:23.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 49 2026-02-20T21:12:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:23.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=198, tag_tid=14, entry_tid=41206], [object_number=197, tag_tid=14, entry_tid=41205], [object_number=196, tag_tid=14, entry_tid=41204], [object_number=199, tag_tid=14, entry_tid=41203]]], state=connected] 2026-02-20T21:12:23.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:23.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 49 2026-02-20T21:12:23.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:23.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:23.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=198, tag_tid=14, entry_tid=41206], [object_number=197, tag_tid=14, entry_tid=41205], [object_number=196, tag_tid=14, entry_tid=41204], [object_number=199, tag_tid=14, entry_tid=41203]]], state=connected] 2026-02-20T21:12:23.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=198, tag_tid=14, entry_tid=41206' 2026-02-20T21:12:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=198, tag_tid=14, entry_tid=41206' 2026-02-20T21:12:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=198, tag_tid=14, entry_tid=41206' '!=' 'object_number=196, tag_tid=14, entry_tid=40892' 2026-02-20T21:12:23.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:23.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=198, tag_tid=14, entry_tid=41206' = 'object_number=196, tag_tid=14, entry_tid=40892' 2026-02-20T21:12:23.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=198, tag_tid=14, entry_tid=41206' 2026-02-20T21:12:23.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:23.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:23.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:23.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:23.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=198, tag_tid=14, entry_tid=41206' 2026-02-20T21:12:23.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:23.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:23.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:23.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:23.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:23.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:23.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=198, tag_tid=14, entry_tid=41206' 2026-02-20T21:12:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:23.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41206 2026-02-20T21:12:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 41206 2026-02-20T21:12:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:23.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:23.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:23.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:23.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:23.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:23.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:23.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:23.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:23.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 49 2026-02-20T21:12:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:23.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=198, tag_tid=14, entry_tid=41542], [object_number=197, tag_tid=14, entry_tid=41541], [object_number=196, tag_tid=14, entry_tid=41540], [object_number=199, tag_tid=14, entry_tid=41539]]], state=connected] 2026-02-20T21:12:23.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:23.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:23.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:23.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:23.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:23.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:23.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:23.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:23.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 49 2026-02-20T21:12:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:23.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=198, tag_tid=14, entry_tid=41542], [object_number=197, tag_tid=14, entry_tid=41541], [object_number=196, tag_tid=14, entry_tid=41540], [object_number=199, tag_tid=14, entry_tid=41539]]], state=connected] 2026-02-20T21:12:23.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=198, tag_tid=14, entry_tid=41542' 2026-02-20T21:12:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=198, tag_tid=14, entry_tid=41542' 2026-02-20T21:12:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=198, tag_tid=14, entry_tid=41542' '!=' 'object_number=198, tag_tid=14, entry_tid=41206' 2026-02-20T21:12:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=198, tag_tid=14, entry_tid=41542' = 'object_number=198, tag_tid=14, entry_tid=41206' 2026-02-20T21:12:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=198, tag_tid=14, entry_tid=41542' 2026-02-20T21:12:23.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:23.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:23.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:23.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:23.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=198, tag_tid=14, entry_tid=41542' 2026-02-20T21:12:23.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:23.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:23.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:23.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:23.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:23.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:23.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:23.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=198, tag_tid=14, entry_tid=41542' 2026-02-20T21:12:23.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:23.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:23.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41542 2026-02-20T21:12:23.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 41542 2026-02-20T21:12:23.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:23.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:23.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:24.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:24.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:24.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:24.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:24.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:24.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:24.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:24.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:24.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:24.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:24.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:24.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:24.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:24.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:24.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:24.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:24.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 49 2026-02-20T21:12:24.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:24.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:24.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:24.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=196, tag_tid=14, entry_tid=41844], [object_number=199, tag_tid=14, entry_tid=41843], [object_number=198, tag_tid=14, entry_tid=41842], [object_number=197, tag_tid=14, entry_tid=41841]]], state=connected] 2026-02-20T21:12:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:24.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:24.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 49 2026-02-20T21:12:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=196, tag_tid=14, entry_tid=41844], [object_number=199, tag_tid=14, entry_tid=41843], [object_number=198, tag_tid=14, entry_tid=41842], [object_number=197, tag_tid=14, entry_tid=41841]]], state=connected] 2026-02-20T21:12:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=196, tag_tid=14, entry_tid=41844' 2026-02-20T21:12:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=196, tag_tid=14, entry_tid=41844' 2026-02-20T21:12:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=196, tag_tid=14, entry_tid=41844' '!=' 'object_number=198, tag_tid=14, entry_tid=41542' 2026-02-20T21:12:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=196, tag_tid=14, entry_tid=41844' = 'object_number=198, tag_tid=14, entry_tid=41542' 2026-02-20T21:12:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=196, tag_tid=14, entry_tid=41844' 2026-02-20T21:12:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=196, tag_tid=14, entry_tid=41844' 2026-02-20T21:12:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:24.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:24.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:24.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=196, tag_tid=14, entry_tid=41844' 2026-02-20T21:12:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41844 2026-02-20T21:12:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 41844 2026-02-20T21:12:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:24.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:24.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:24.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:24.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:24.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:24.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:24.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:24.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:24.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:24.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:24.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:24.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:24.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:24.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:24.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:24.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:24.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:24.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:24.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:24.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:24.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:24.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:24.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:24.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:24.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:24.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:24.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=200, tag_tid=14, entry_tid=42096], [object_number=203, tag_tid=14, entry_tid=42095], [object_number=202, tag_tid=14, entry_tid=42094], [object_number=201, tag_tid=14, entry_tid=42093]]], state=connected] 2026-02-20T21:12:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:24.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:24.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:24.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:24.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=200, tag_tid=14, entry_tid=42096], [object_number=203, tag_tid=14, entry_tid=42095], [object_number=202, tag_tid=14, entry_tid=42094], [object_number=201, tag_tid=14, entry_tid=42093]]], state=connected] 2026-02-20T21:12:24.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=200, tag_tid=14, entry_tid=42096' 2026-02-20T21:12:24.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=200, tag_tid=14, entry_tid=42096' 2026-02-20T21:12:24.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=200, tag_tid=14, entry_tid=42096' '!=' 'object_number=196, tag_tid=14, entry_tid=41844' 2026-02-20T21:12:24.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:24.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=200, tag_tid=14, entry_tid=42096' = 'object_number=196, tag_tid=14, entry_tid=41844' 2026-02-20T21:12:24.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=200, tag_tid=14, entry_tid=42096' 2026-02-20T21:12:24.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:24.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:24.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:24.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:24.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=200, tag_tid=14, entry_tid=42096' 2026-02-20T21:12:24.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:24.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:24.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:24.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:24.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:24.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:24.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:24.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=200, tag_tid=14, entry_tid=42096' 2026-02-20T21:12:24.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:24.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:24.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42096 2026-02-20T21:12:24.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 42096 2026-02-20T21:12:24.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:24.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:24.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:24.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:24.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:24.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:24.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:24.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:24.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:24.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:24.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:24.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:24.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:24.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=200, tag_tid=14, entry_tid=42420], [object_number=203, tag_tid=14, entry_tid=42419], [object_number=202, tag_tid=14, entry_tid=42418], [object_number=201, tag_tid=14, entry_tid=42417]]], state=connected] 2026-02-20T21:12:24.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:24.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:24.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:24.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:24.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:24.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:24.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:24.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:24.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:24.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:24.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:24.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:24.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:24.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:24.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:24.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=200, tag_tid=14, entry_tid=42420], [object_number=203, tag_tid=14, entry_tid=42419], [object_number=202, tag_tid=14, entry_tid=42418], [object_number=201, tag_tid=14, entry_tid=42417]]], state=connected] 2026-02-20T21:12:24.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=200, tag_tid=14, entry_tid=42420' 2026-02-20T21:12:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=200, tag_tid=14, entry_tid=42420' 2026-02-20T21:12:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=200, tag_tid=14, entry_tid=42420' '!=' 'object_number=200, tag_tid=14, entry_tid=42096' 2026-02-20T21:12:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=200, tag_tid=14, entry_tid=42420' = 'object_number=200, tag_tid=14, entry_tid=42096' 2026-02-20T21:12:24.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=200, tag_tid=14, entry_tid=42420' 2026-02-20T21:12:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=200, tag_tid=14, entry_tid=42420' 2026-02-20T21:12:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:24.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:24.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=200, tag_tid=14, entry_tid=42420' 2026-02-20T21:12:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42420 2026-02-20T21:12:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 42420 2026-02-20T21:12:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:24.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:25.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:25.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:25.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=202, tag_tid=14, entry_tid=42714], [object_number=201, tag_tid=14, entry_tid=42713], [object_number=200, tag_tid=14, entry_tid=42712], [object_number=203, tag_tid=14, entry_tid=42711]]], state=connected] 2026-02-20T21:12:25.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:25.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:25.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:25.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:25.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=202, tag_tid=14, entry_tid=42714], [object_number=201, tag_tid=14, entry_tid=42713], [object_number=200, tag_tid=14, entry_tid=42712], [object_number=203, tag_tid=14, entry_tid=42711]]], state=connected] 2026-02-20T21:12:25.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=202, tag_tid=14, entry_tid=42714' 2026-02-20T21:12:25.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=202, tag_tid=14, entry_tid=42714' 2026-02-20T21:12:25.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=202, tag_tid=14, entry_tid=42714' '!=' 'object_number=200, tag_tid=14, entry_tid=42420' 2026-02-20T21:12:25.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=202, tag_tid=14, entry_tid=42714' = 'object_number=200, tag_tid=14, entry_tid=42420' 2026-02-20T21:12:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=202, tag_tid=14, entry_tid=42714' 2026-02-20T21:12:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=202, tag_tid=14, entry_tid=42714' 2026-02-20T21:12:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:25.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=202, tag_tid=14, entry_tid=42714' 2026-02-20T21:12:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42714 2026-02-20T21:12:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 42714 2026-02-20T21:12:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:25.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:25.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:25.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:25.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:25.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:25.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:25.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:25.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:25.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:25.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:25.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:25.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=200, tag_tid=14, entry_tid=43036], [object_number=203, tag_tid=14, entry_tid=43035], [object_number=202, tag_tid=14, entry_tid=43034], [object_number=201, tag_tid=14, entry_tid=43033]]], state=connected] 2026-02-20T21:12:25.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:25.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:25.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:25.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:25.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:25.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:25.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:25.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:25.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:25.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:25.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:25.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:25.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:25.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:25.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:25.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:25.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:25.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:25.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=200, tag_tid=14, entry_tid=43036], [object_number=203, tag_tid=14, entry_tid=43035], [object_number=202, tag_tid=14, entry_tid=43034], [object_number=201, tag_tid=14, entry_tid=43033]]], state=connected] 2026-02-20T21:12:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=200, tag_tid=14, entry_tid=43036' 2026-02-20T21:12:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=200, tag_tid=14, entry_tid=43036' 2026-02-20T21:12:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=200, tag_tid=14, entry_tid=43036' '!=' 'object_number=202, tag_tid=14, entry_tid=42714' 2026-02-20T21:12:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=200, tag_tid=14, entry_tid=43036' = 'object_number=202, tag_tid=14, entry_tid=42714' 2026-02-20T21:12:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=200, tag_tid=14, entry_tid=43036' 2026-02-20T21:12:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:25.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:25.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:25.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=200, tag_tid=14, entry_tid=43036' 2026-02-20T21:12:25.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:25.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:25.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:25.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:25.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:25.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:25.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=200, tag_tid=14, entry_tid=43036' 2026-02-20T21:12:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:25.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43036 2026-02-20T21:12:25.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 43036 2026-02-20T21:12:25.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:25.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:25.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:25.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:25.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:25.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:25.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:25.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:25.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:25.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:25.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:25.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:25.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:25.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:25.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:25.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:25.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:25.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:25.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:25.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:25.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=200, tag_tid=14, entry_tid=43308], [object_number=203, tag_tid=14, entry_tid=43307], [object_number=202, tag_tid=14, entry_tid=43306], [object_number=201, tag_tid=14, entry_tid=43305]]], state=connected] 2026-02-20T21:12:25.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:25.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=200, tag_tid=14, entry_tid=43308], [object_number=203, tag_tid=14, entry_tid=43307], [object_number=202, tag_tid=14, entry_tid=43306], [object_number=201, tag_tid=14, entry_tid=43305]]], state=connected] 2026-02-20T21:12:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=200, tag_tid=14, entry_tid=43308' 2026-02-20T21:12:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=200, tag_tid=14, entry_tid=43308' 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=200, tag_tid=14, entry_tid=43308' '!=' 'object_number=200, tag_tid=14, entry_tid=43036' 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=200, tag_tid=14, entry_tid=43308' = 'object_number=200, tag_tid=14, entry_tid=43036' 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=200, tag_tid=14, entry_tid=43308' 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=200, tag_tid=14, entry_tid=43308' 2026-02-20T21:12:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:26.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:26.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:26.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:26.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:26.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=200, tag_tid=14, entry_tid=43308' 2026-02-20T21:12:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43308 2026-02-20T21:12:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 43308 2026-02-20T21:12:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:26.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:26.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:26.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:26.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:26.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:26.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:26.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:26.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:26.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:26.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:26.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:26.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:26.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=201, tag_tid=14, entry_tid=43801], [object_number=200, tag_tid=14, entry_tid=43800], [object_number=203, tag_tid=14, entry_tid=43799], [object_number=202, tag_tid=14, entry_tid=43798]]], state=connected] 2026-02-20T21:12:26.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:26.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=201, tag_tid=14, entry_tid=43801], [object_number=200, tag_tid=14, entry_tid=43800], [object_number=203, tag_tid=14, entry_tid=43799], [object_number=202, tag_tid=14, entry_tid=43798]]], state=connected] 2026-02-20T21:12:26.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=201, tag_tid=14, entry_tid=43801' 2026-02-20T21:12:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=201, tag_tid=14, entry_tid=43801' 2026-02-20T21:12:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=201, tag_tid=14, entry_tid=43801' '!=' 'object_number=200, tag_tid=14, entry_tid=43308' 2026-02-20T21:12:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=201, tag_tid=14, entry_tid=43801' = 'object_number=200, tag_tid=14, entry_tid=43308' 2026-02-20T21:12:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=201, tag_tid=14, entry_tid=43801' 2026-02-20T21:12:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=201, tag_tid=14, entry_tid=43801' 2026-02-20T21:12:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:26.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:26.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:26.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:26.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:26.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=201, tag_tid=14, entry_tid=43801' 2026-02-20T21:12:26.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:26.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43801 2026-02-20T21:12:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 43801 2026-02-20T21:12:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:26.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:26.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:26.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:26.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:26.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:26.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:26.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:26.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:26.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:26.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:26.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:26.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:26.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:26.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:26.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:26.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:26.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:26.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:26.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:26.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:26.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:26.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:26.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:26.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:26.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:26.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:26.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:26.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=202, tag_tid=14, entry_tid=44138], [object_number=201, tag_tid=14, entry_tid=44137], [object_number=200, tag_tid=14, entry_tid=44136], [object_number=203, tag_tid=14, entry_tid=44135]]], state=connected] 2026-02-20T21:12:26.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:26.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:26.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:26.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:26.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:26.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:26.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=202, tag_tid=14, entry_tid=44138], [object_number=201, tag_tid=14, entry_tid=44137], [object_number=200, tag_tid=14, entry_tid=44136], [object_number=203, tag_tid=14, entry_tid=44135]]], state=connected] 2026-02-20T21:12:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=202, tag_tid=14, entry_tid=44138' 2026-02-20T21:12:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=202, tag_tid=14, entry_tid=44138' 2026-02-20T21:12:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=202, tag_tid=14, entry_tid=44138' '!=' 'object_number=201, tag_tid=14, entry_tid=43801' 2026-02-20T21:12:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=202, tag_tid=14, entry_tid=44138' = 'object_number=201, tag_tid=14, entry_tid=43801' 2026-02-20T21:12:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=202, tag_tid=14, entry_tid=44138' 2026-02-20T21:12:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=202, tag_tid=14, entry_tid=44138' 2026-02-20T21:12:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:26.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:26.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:26.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=202, tag_tid=14, entry_tid=44138' 2026-02-20T21:12:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:26.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:26.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44138 2026-02-20T21:12:26.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 44138 2026-02-20T21:12:26.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:26.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:26.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:26.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:26.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:27.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:27.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:27.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=201, tag_tid=14, entry_tid=44457], [object_number=200, tag_tid=14, entry_tid=44456], [object_number=203, tag_tid=14, entry_tid=44455], [object_number=202, tag_tid=14, entry_tid=44454]]], state=connected] 2026-02-20T21:12:27.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:27.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:27.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:27.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:27.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:27.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:27.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=201, tag_tid=14, entry_tid=44457], [object_number=200, tag_tid=14, entry_tid=44456], [object_number=203, tag_tid=14, entry_tid=44455], [object_number=202, tag_tid=14, entry_tid=44454]]], state=connected] 2026-02-20T21:12:27.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=201, tag_tid=14, entry_tid=44457' 2026-02-20T21:12:27.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=201, tag_tid=14, entry_tid=44457' 2026-02-20T21:12:27.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=201, tag_tid=14, entry_tid=44457' '!=' 'object_number=202, tag_tid=14, entry_tid=44138' 2026-02-20T21:12:27.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:27.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=201, tag_tid=14, entry_tid=44457' = 'object_number=202, tag_tid=14, entry_tid=44138' 2026-02-20T21:12:27.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=201, tag_tid=14, entry_tid=44457' 2026-02-20T21:12:27.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:27.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:27.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:27.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:27.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=201, tag_tid=14, entry_tid=44457' 2026-02-20T21:12:27.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:27.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:27.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:27.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:27.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:27.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:27.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=201, tag_tid=14, entry_tid=44457' 2026-02-20T21:12:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44457 2026-02-20T21:12:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 44457 2026-02-20T21:12:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:27.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:27.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:27.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:27.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:27.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:27.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:27.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:27.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:27.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:27.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:27.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:27.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:27.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:27.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:27.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:27.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:27.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:27.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:27.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=201, tag_tid=14, entry_tid=44969], [object_number=200, tag_tid=14, entry_tid=44968], [object_number=203, tag_tid=14, entry_tid=44967], [object_number=202, tag_tid=14, entry_tid=44966]]], state=connected] 2026-02-20T21:12:27.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:27.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:27.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:27.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:27.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:27.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=201, tag_tid=14, entry_tid=44969], [object_number=200, tag_tid=14, entry_tid=44968], [object_number=203, tag_tid=14, entry_tid=44967], [object_number=202, tag_tid=14, entry_tid=44966]]], state=connected] 2026-02-20T21:12:27.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=201, tag_tid=14, entry_tid=44969' 2026-02-20T21:12:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=201, tag_tid=14, entry_tid=44969' 2026-02-20T21:12:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=201, tag_tid=14, entry_tid=44969' '!=' 'object_number=201, tag_tid=14, entry_tid=44457' 2026-02-20T21:12:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=201, tag_tid=14, entry_tid=44969' = 'object_number=201, tag_tid=14, entry_tid=44457' 2026-02-20T21:12:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=201, tag_tid=14, entry_tid=44969' 2026-02-20T21:12:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=201, tag_tid=14, entry_tid=44969' 2026-02-20T21:12:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=201, tag_tid=14, entry_tid=44969' 2026-02-20T21:12:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44969 2026-02-20T21:12:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 44969 2026-02-20T21:12:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:27.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:27.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:27.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:27.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:27.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:27.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:27.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:27.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=203, tag_tid=14, entry_tid=45283], [object_number=202, tag_tid=14, entry_tid=45282], [object_number=201, tag_tid=14, entry_tid=45281], [object_number=200, tag_tid=14, entry_tid=45280]]], state=connected] 2026-02-20T21:12:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=203, tag_tid=14, entry_tid=45283], [object_number=202, tag_tid=14, entry_tid=45282], [object_number=201, tag_tid=14, entry_tid=45281], [object_number=200, tag_tid=14, entry_tid=45280]]], state=connected] 2026-02-20T21:12:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=203, tag_tid=14, entry_tid=45283' 2026-02-20T21:12:27.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=203, tag_tid=14, entry_tid=45283' 2026-02-20T21:12:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=203, tag_tid=14, entry_tid=45283' '!=' 'object_number=201, tag_tid=14, entry_tid=44969' 2026-02-20T21:12:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=203, tag_tid=14, entry_tid=45283' = 'object_number=201, tag_tid=14, entry_tid=44969' 2026-02-20T21:12:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=203, tag_tid=14, entry_tid=45283' 2026-02-20T21:12:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:28.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=203, tag_tid=14, entry_tid=45283' 2026-02-20T21:12:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:28.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=203, tag_tid=14, entry_tid=45283' 2026-02-20T21:12:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45283 2026-02-20T21:12:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 45283 2026-02-20T21:12:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:28.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:28.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:28.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:28.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:28.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:28.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:28.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:28.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:28.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:28.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:28.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:28.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:28.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:28.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=202, tag_tid=14, entry_tid=45634], [object_number=201, tag_tid=14, entry_tid=45633], [object_number=200, tag_tid=14, entry_tid=45632], [object_number=203, tag_tid=14, entry_tid=45631]]], state=connected] 2026-02-20T21:12:28.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:28.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:28.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:28.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:28.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=202, tag_tid=14, entry_tid=45634], [object_number=201, tag_tid=14, entry_tid=45633], [object_number=200, tag_tid=14, entry_tid=45632], [object_number=203, tag_tid=14, entry_tid=45631]]], state=connected] 2026-02-20T21:12:28.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=202, tag_tid=14, entry_tid=45634' 2026-02-20T21:12:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=202, tag_tid=14, entry_tid=45634' 2026-02-20T21:12:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=202, tag_tid=14, entry_tid=45634' '!=' 'object_number=203, tag_tid=14, entry_tid=45283' 2026-02-20T21:12:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=202, tag_tid=14, entry_tid=45634' = 'object_number=203, tag_tid=14, entry_tid=45283' 2026-02-20T21:12:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=202, tag_tid=14, entry_tid=45634' 2026-02-20T21:12:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:28.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=202, tag_tid=14, entry_tid=45634' 2026-02-20T21:12:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=202, tag_tid=14, entry_tid=45634' 2026-02-20T21:12:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45634 2026-02-20T21:12:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 45634 2026-02-20T21:12:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:28.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:28.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:28.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:28.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:28.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:28.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:28.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:28.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:28.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:28.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:28.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:28.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:28.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:28.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:28.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:28.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:28.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:28.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:28.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:28.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:28.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:28.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:28.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:28.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:28.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:28.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:28.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:28.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:28.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:28.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:28.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:28.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:28.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:28.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:28.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:28.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:28.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:28.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:28.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:28.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:28.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:28.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:28.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=202, tag_tid=14, entry_tid=45998], [object_number=201, tag_tid=14, entry_tid=45997], [object_number=200, tag_tid=14, entry_tid=45996], [object_number=203, tag_tid=14, entry_tid=45995]]], state=connected] 2026-02-20T21:12:28.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:28.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:28.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:28.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:28.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 50 2026-02-20T21:12:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:28.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:28.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:28.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=202, tag_tid=14, entry_tid=45998], [object_number=201, tag_tid=14, entry_tid=45997], [object_number=200, tag_tid=14, entry_tid=45996], [object_number=203, tag_tid=14, entry_tid=45995]]], state=connected] 2026-02-20T21:12:28.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=202, tag_tid=14, entry_tid=45998' 2026-02-20T21:12:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=202, tag_tid=14, entry_tid=45998' 2026-02-20T21:12:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=202, tag_tid=14, entry_tid=45998' '!=' 'object_number=202, tag_tid=14, entry_tid=45634' 2026-02-20T21:12:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=202, tag_tid=14, entry_tid=45998' = 'object_number=202, tag_tid=14, entry_tid=45634' 2026-02-20T21:12:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=202, tag_tid=14, entry_tid=45998' 2026-02-20T21:12:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=202, tag_tid=14, entry_tid=45998' 2026-02-20T21:12:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:28.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:28.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:28.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:28.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:28.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:28.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=202, tag_tid=14, entry_tid=45998' 2026-02-20T21:12:28.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:28.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45998 2026-02-20T21:12:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 45998 2026-02-20T21:12:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:28.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:28.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:28.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:28.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:28.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:28.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:28.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:28.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:28.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:28.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:28.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:28.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:28.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:28.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:28.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:28.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:28.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:28.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:28.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:28.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=46227], [object_number=206, tag_tid=14, entry_tid=46226], [object_number=205, tag_tid=14, entry_tid=46225], [object_number=204, tag_tid=14, entry_tid=46224]]], state=connected] 2026-02-20T21:12:28.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:28.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:28.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:28.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:28.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:28.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:28.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:29.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=46227], [object_number=206, tag_tid=14, entry_tid=46226], [object_number=205, tag_tid=14, entry_tid=46225], [object_number=204, tag_tid=14, entry_tid=46224]]], state=connected] 2026-02-20T21:12:29.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=207, tag_tid=14, entry_tid=46227' 2026-02-20T21:12:29.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=207, tag_tid=14, entry_tid=46227' 2026-02-20T21:12:29.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=46227' '!=' 'object_number=202, tag_tid=14, entry_tid=45998' 2026-02-20T21:12:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=46227' = 'object_number=202, tag_tid=14, entry_tid=45998' 2026-02-20T21:12:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=207, tag_tid=14, entry_tid=46227' 2026-02-20T21:12:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:29.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:29.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=46227' 2026-02-20T21:12:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:29.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:29.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:29.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:29.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:29.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=46227' 2026-02-20T21:12:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:29.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46227 2026-02-20T21:12:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 46227 2026-02-20T21:12:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:29.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:29.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:29.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:29.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:29.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:29.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:29.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:29.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:29.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:29.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:29.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:29.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:29.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:29.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:29.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:29.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:29.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:29.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=205, tag_tid=14, entry_tid=46537], [object_number=204, tag_tid=14, entry_tid=46536], [object_number=207, tag_tid=14, entry_tid=46535], [object_number=206, tag_tid=14, entry_tid=46534]]], state=connected] 2026-02-20T21:12:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:29.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:29.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:29.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:29.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:29.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=205, tag_tid=14, entry_tid=46537], [object_number=204, tag_tid=14, entry_tid=46536], [object_number=207, tag_tid=14, entry_tid=46535], [object_number=206, tag_tid=14, entry_tid=46534]]], state=connected] 2026-02-20T21:12:29.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=205, tag_tid=14, entry_tid=46537' 2026-02-20T21:12:29.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=205, tag_tid=14, entry_tid=46537' 2026-02-20T21:12:29.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=205, tag_tid=14, entry_tid=46537' '!=' 'object_number=207, tag_tid=14, entry_tid=46227' 2026-02-20T21:12:29.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:29.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=205, tag_tid=14, entry_tid=46537' = 'object_number=207, tag_tid=14, entry_tid=46227' 2026-02-20T21:12:29.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=205, tag_tid=14, entry_tid=46537' 2026-02-20T21:12:29.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:29.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:29.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:29.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=205, tag_tid=14, entry_tid=46537' 2026-02-20T21:12:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:29.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:29.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:29.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=205, tag_tid=14, entry_tid=46537' 2026-02-20T21:12:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46537 2026-02-20T21:12:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 46537 2026-02-20T21:12:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:29.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:29.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:29.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:29.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:29.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:29.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:29.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:29.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:29.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:29.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:29.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:29.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:29.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:29.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:29.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:29.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:29.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:29.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:29.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:29.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:29.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:29.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:29.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:29.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:29.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:29.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:29.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:29.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:29.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=47074], [object_number=205, tag_tid=14, entry_tid=47073], [object_number=204, tag_tid=14, entry_tid=47072], [object_number=207, tag_tid=14, entry_tid=47071]]], state=connected] 2026-02-20T21:12:29.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:29.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:29.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:29.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:29.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:29.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:29.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:29.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:29.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:29.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:29.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:29.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=47074], [object_number=205, tag_tid=14, entry_tid=47073], [object_number=204, tag_tid=14, entry_tid=47072], [object_number=207, tag_tid=14, entry_tid=47071]]], state=connected] 2026-02-20T21:12:29.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=206, tag_tid=14, entry_tid=47074' 2026-02-20T21:12:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=206, tag_tid=14, entry_tid=47074' 2026-02-20T21:12:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=47074' '!=' 'object_number=205, tag_tid=14, entry_tid=46537' 2026-02-20T21:12:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=47074' = 'object_number=205, tag_tid=14, entry_tid=46537' 2026-02-20T21:12:29.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=206, tag_tid=14, entry_tid=47074' 2026-02-20T21:12:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:29.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=47074' 2026-02-20T21:12:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:29.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:29.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:29.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:29.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:29.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=47074' 2026-02-20T21:12:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47074 2026-02-20T21:12:29.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 47074 2026-02-20T21:12:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:30.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:30.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:30.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:30.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:30.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:30.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:30.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:30.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:30.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:30.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:30.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:30.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=204, tag_tid=14, entry_tid=47440], [object_number=207, tag_tid=14, entry_tid=47439], [object_number=206, tag_tid=14, entry_tid=47438], [object_number=205, tag_tid=14, entry_tid=47437]]], state=connected] 2026-02-20T21:12:30.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:30.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:30.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:30.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:30.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:30.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:30.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:30.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:30.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:30.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:30.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:30.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:30.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:30.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:30.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=204, tag_tid=14, entry_tid=47440], [object_number=207, tag_tid=14, entry_tid=47439], [object_number=206, tag_tid=14, entry_tid=47438], [object_number=205, tag_tid=14, entry_tid=47437]]], state=connected] 2026-02-20T21:12:30.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:30.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=204, tag_tid=14, entry_tid=47440' 2026-02-20T21:12:30.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=204, tag_tid=14, entry_tid=47440' 2026-02-20T21:12:30.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=204, tag_tid=14, entry_tid=47440' '!=' 'object_number=206, tag_tid=14, entry_tid=47074' 2026-02-20T21:12:30.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:30.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=204, tag_tid=14, entry_tid=47440' = 'object_number=206, tag_tid=14, entry_tid=47074' 2026-02-20T21:12:30.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=204, tag_tid=14, entry_tid=47440' 2026-02-20T21:12:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:30.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=204, tag_tid=14, entry_tid=47440' 2026-02-20T21:12:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:30.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:30.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:30.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:30.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:30.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:30.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:30.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=204, tag_tid=14, entry_tid=47440' 2026-02-20T21:12:30.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:30.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:30.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47440 2026-02-20T21:12:30.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 47440 2026-02-20T21:12:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:30.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:30.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=47778], [object_number=205, tag_tid=14, entry_tid=47777], [object_number=204, tag_tid=14, entry_tid=47776], [object_number=207, tag_tid=14, entry_tid=47775]]], state=connected] 2026-02-20T21:12:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:30.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:30.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:30.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:30.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:30.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:30.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=47778], [object_number=205, tag_tid=14, entry_tid=47777], [object_number=204, tag_tid=14, entry_tid=47776], [object_number=207, tag_tid=14, entry_tid=47775]]], state=connected] 2026-02-20T21:12:30.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=206, tag_tid=14, entry_tid=47778' 2026-02-20T21:12:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=206, tag_tid=14, entry_tid=47778' 2026-02-20T21:12:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=47778' '!=' 'object_number=204, tag_tid=14, entry_tid=47440' 2026-02-20T21:12:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=47778' = 'object_number=204, tag_tid=14, entry_tid=47440' 2026-02-20T21:12:30.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=206, tag_tid=14, entry_tid=47778' 2026-02-20T21:12:30.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:30.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:30.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:30.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=47778' 2026-02-20T21:12:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:30.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:30.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:30.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=47778' 2026-02-20T21:12:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47778 2026-02-20T21:12:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 47778 2026-02-20T21:12:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:30.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:30.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:30.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:30.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:31.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:31.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:31.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:31.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:31.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:31.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:31.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:31.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=205, tag_tid=14, entry_tid=48253], [object_number=204, tag_tid=14, entry_tid=48252], [object_number=207, tag_tid=14, entry_tid=48251], [object_number=206, tag_tid=14, entry_tid=48250]]], state=connected] 2026-02-20T21:12:31.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:31.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:31.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:31.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:31.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:31.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:31.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=205, tag_tid=14, entry_tid=48253], [object_number=204, tag_tid=14, entry_tid=48252], [object_number=207, tag_tid=14, entry_tid=48251], [object_number=206, tag_tid=14, entry_tid=48250]]], state=connected] 2026-02-20T21:12:31.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=205, tag_tid=14, entry_tid=48253' 2026-02-20T21:12:31.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=205, tag_tid=14, entry_tid=48253' 2026-02-20T21:12:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=205, tag_tid=14, entry_tid=48253' '!=' 'object_number=206, tag_tid=14, entry_tid=47778' 2026-02-20T21:12:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=205, tag_tid=14, entry_tid=48253' = 'object_number=206, tag_tid=14, entry_tid=47778' 2026-02-20T21:12:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=205, tag_tid=14, entry_tid=48253' 2026-02-20T21:12:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:31.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=205, tag_tid=14, entry_tid=48253' 2026-02-20T21:12:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:31.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=205, tag_tid=14, entry_tid=48253' 2026-02-20T21:12:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48253 2026-02-20T21:12:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 48253 2026-02-20T21:12:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:31.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:31.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:31.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:31.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:31.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:31.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:31.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:31.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:31.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:31.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:31.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:31.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:31.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:31.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:31.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:31.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:31.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:31.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=204, tag_tid=14, entry_tid=48564], [object_number=207, tag_tid=14, entry_tid=48563], [object_number=206, tag_tid=14, entry_tid=48562], [object_number=205, tag_tid=14, entry_tid=48561]]], state=connected] 2026-02-20T21:12:31.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:31.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:31.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:31.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:31.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:31.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:31.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=204, tag_tid=14, entry_tid=48564], [object_number=207, tag_tid=14, entry_tid=48563], [object_number=206, tag_tid=14, entry_tid=48562], [object_number=205, tag_tid=14, entry_tid=48561]]], state=connected] 2026-02-20T21:12:31.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=204, tag_tid=14, entry_tid=48564' 2026-02-20T21:12:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=204, tag_tid=14, entry_tid=48564' 2026-02-20T21:12:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=204, tag_tid=14, entry_tid=48564' '!=' 'object_number=205, tag_tid=14, entry_tid=48253' 2026-02-20T21:12:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:31.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=204, tag_tid=14, entry_tid=48564' = 'object_number=205, tag_tid=14, entry_tid=48253' 2026-02-20T21:12:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=204, tag_tid=14, entry_tid=48564' 2026-02-20T21:12:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:31.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=204, tag_tid=14, entry_tid=48564' 2026-02-20T21:12:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:31.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:31.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:31.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=204, tag_tid=14, entry_tid=48564' 2026-02-20T21:12:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:31.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:31.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48564 2026-02-20T21:12:31.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 48564 2026-02-20T21:12:31.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:31.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:31.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:31.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:31.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:31.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:31.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:31.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:31.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:31.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:31.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:31.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:31.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:31.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:31.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:31.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:31.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:31.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:31.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:31.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:31.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:31.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=48862], [object_number=205, tag_tid=14, entry_tid=48861], [object_number=204, tag_tid=14, entry_tid=48860], [object_number=207, tag_tid=14, entry_tid=48859]]], state=connected] 2026-02-20T21:12:31.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:31.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:31.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:31.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:31.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:31.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:31.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:31.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:31.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:31.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:31.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:31.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:31.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:31.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:31.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=48862], [object_number=205, tag_tid=14, entry_tid=48861], [object_number=204, tag_tid=14, entry_tid=48860], [object_number=207, tag_tid=14, entry_tid=48859]]], state=connected] 2026-02-20T21:12:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=206, tag_tid=14, entry_tid=48862' 2026-02-20T21:12:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=206, tag_tid=14, entry_tid=48862' 2026-02-20T21:12:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=48862' '!=' 'object_number=204, tag_tid=14, entry_tid=48564' 2026-02-20T21:12:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=48862' = 'object_number=204, tag_tid=14, entry_tid=48564' 2026-02-20T21:12:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=206, tag_tid=14, entry_tid=48862' 2026-02-20T21:12:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:31.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=48862' 2026-02-20T21:12:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=48862' 2026-02-20T21:12:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:31.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48862 2026-02-20T21:12:31.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 48862 2026-02-20T21:12:31.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:31.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:31.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:32.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:32.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:32.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:32.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:32.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:32.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:32.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=49182], [object_number=205, tag_tid=14, entry_tid=49181], [object_number=204, tag_tid=14, entry_tid=49180], [object_number=207, tag_tid=14, entry_tid=49179]]], state=connected] 2026-02-20T21:12:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:32.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:32.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:32.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=49182], [object_number=205, tag_tid=14, entry_tid=49181], [object_number=204, tag_tid=14, entry_tid=49180], [object_number=207, tag_tid=14, entry_tid=49179]]], state=connected] 2026-02-20T21:12:32.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=206, tag_tid=14, entry_tid=49182' 2026-02-20T21:12:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=206, tag_tid=14, entry_tid=49182' 2026-02-20T21:12:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=49182' '!=' 'object_number=206, tag_tid=14, entry_tid=48862' 2026-02-20T21:12:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=49182' = 'object_number=206, tag_tid=14, entry_tid=48862' 2026-02-20T21:12:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=206, tag_tid=14, entry_tid=49182' 2026-02-20T21:12:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:32.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:32.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=49182' 2026-02-20T21:12:32.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:32.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:32.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:32.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:32.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:32.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:32.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:32.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=49182' 2026-02-20T21:12:32.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:32.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:32.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49182 2026-02-20T21:12:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 49182 2026-02-20T21:12:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:32.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:32.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:32.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:32.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:32.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:32.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:32.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:32.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:32.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:32.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:32.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:32.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:32.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:32.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=49502], [object_number=205, tag_tid=14, entry_tid=49501], [object_number=204, tag_tid=14, entry_tid=49500], [object_number=207, tag_tid=14, entry_tid=49499]]], state=connected] 2026-02-20T21:12:32.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:32.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:32.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:32.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:32.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:32.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:32.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:32.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:32.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:32.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:32.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:32.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:32.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:32.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:32.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=49502], [object_number=205, tag_tid=14, entry_tid=49501], [object_number=204, tag_tid=14, entry_tid=49500], [object_number=207, tag_tid=14, entry_tid=49499]]], state=connected] 2026-02-20T21:12:32.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=206, tag_tid=14, entry_tid=49502' 2026-02-20T21:12:32.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=206, tag_tid=14, entry_tid=49502' 2026-02-20T21:12:32.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=49502' '!=' 'object_number=206, tag_tid=14, entry_tid=49182' 2026-02-20T21:12:32.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:32.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=49502' = 'object_number=206, tag_tid=14, entry_tid=49182' 2026-02-20T21:12:32.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=206, tag_tid=14, entry_tid=49502' 2026-02-20T21:12:32.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:32.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:32.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:32.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:32.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=49502' 2026-02-20T21:12:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=49502' 2026-02-20T21:12:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49502 2026-02-20T21:12:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 49502 2026-02-20T21:12:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:32.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:32.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:32.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:32.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:32.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:32.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:32.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:32.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:32.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=49994], [object_number=205, tag_tid=14, entry_tid=49993], [object_number=204, tag_tid=14, entry_tid=49992], [object_number=207, tag_tid=14, entry_tid=49991]]], state=connected] 2026-02-20T21:12:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:33.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:33.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:33.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:33.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=206, tag_tid=14, entry_tid=49994], [object_number=205, tag_tid=14, entry_tid=49993], [object_number=204, tag_tid=14, entry_tid=49992], [object_number=207, tag_tid=14, entry_tid=49991]]], state=connected] 2026-02-20T21:12:33.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=206, tag_tid=14, entry_tid=49994' 2026-02-20T21:12:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=206, tag_tid=14, entry_tid=49994' 2026-02-20T21:12:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=49994' '!=' 'object_number=206, tag_tid=14, entry_tid=49502' 2026-02-20T21:12:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=206, tag_tid=14, entry_tid=49994' = 'object_number=206, tag_tid=14, entry_tid=49502' 2026-02-20T21:12:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=206, tag_tid=14, entry_tid=49994' 2026-02-20T21:12:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=49994' 2026-02-20T21:12:33.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:33.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=206, tag_tid=14, entry_tid=49994' 2026-02-20T21:12:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49994 2026-02-20T21:12:33.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 49994 2026-02-20T21:12:33.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:33.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:33.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:33.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:33.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:33.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:33.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:33.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:33.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:33.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:33.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:33.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:33.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:33.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:33.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:33.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:33.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:33.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:33.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:33.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:33.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:33.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:33.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:33.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:33.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:33.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:33.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=50303], [object_number=206, tag_tid=14, entry_tid=50302], [object_number=205, tag_tid=14, entry_tid=50301], [object_number=204, tag_tid=14, entry_tid=50300]]], state=connected] 2026-02-20T21:12:33.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:33.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:33.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:33.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:33.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:33.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:33.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:33.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:33.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:33.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:33.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:33.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:33.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:33.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:33.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 51 2026-02-20T21:12:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:33.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=207, tag_tid=14, entry_tid=50303], [object_number=206, tag_tid=14, entry_tid=50302], [object_number=205, tag_tid=14, entry_tid=50301], [object_number=204, tag_tid=14, entry_tid=50300]]], state=connected] 2026-02-20T21:12:33.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=207, tag_tid=14, entry_tid=50303' 2026-02-20T21:12:33.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=207, tag_tid=14, entry_tid=50303' 2026-02-20T21:12:33.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=50303' '!=' 'object_number=206, tag_tid=14, entry_tid=49994' 2026-02-20T21:12:33.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:33.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=207, tag_tid=14, entry_tid=50303' = 'object_number=206, tag_tid=14, entry_tid=49994' 2026-02-20T21:12:33.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=207, tag_tid=14, entry_tid=50303' 2026-02-20T21:12:33.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:33.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:33.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:33.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:33.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=50303' 2026-02-20T21:12:33.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:33.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:33.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:33.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:33.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:33.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:33.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=207, tag_tid=14, entry_tid=50303' 2026-02-20T21:12:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:33.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50303 2026-02-20T21:12:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 50303 2026-02-20T21:12:33.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:33.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:33.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:33.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:33.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:33.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:33.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:33.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:33.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:33.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:33.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:33.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:33.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:33.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:33.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:33.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=50530], [object_number=209, tag_tid=14, entry_tid=50529], [object_number=208, tag_tid=14, entry_tid=50528], [object_number=211, tag_tid=14, entry_tid=50527]]], state=connected] 2026-02-20T21:12:33.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:33.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:33.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:33.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:33.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:33.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:33.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:33.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:33.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=50530], [object_number=209, tag_tid=14, entry_tid=50529], [object_number=208, tag_tid=14, entry_tid=50528], [object_number=211, tag_tid=14, entry_tid=50527]]], state=connected] 2026-02-20T21:12:33.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=210, tag_tid=14, entry_tid=50530' 2026-02-20T21:12:33.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=210, tag_tid=14, entry_tid=50530' 2026-02-20T21:12:33.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=50530' '!=' 'object_number=207, tag_tid=14, entry_tid=50303' 2026-02-20T21:12:33.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:33.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=50530' = 'object_number=207, tag_tid=14, entry_tid=50303' 2026-02-20T21:12:33.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=210, tag_tid=14, entry_tid=50530' 2026-02-20T21:12:33.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:33.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:33.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:33.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=50530' 2026-02-20T21:12:33.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:33.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:33.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:33.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:33.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:33.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:33.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:33.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=50530' 2026-02-20T21:12:33.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:33.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50530 2026-02-20T21:12:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 50530 2026-02-20T21:12:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:33.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:33.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:33.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:33.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:33.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:33.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:33.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:33.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:33.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:33.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:33.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:34.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:34.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:34.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:34.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=50829], [object_number=208, tag_tid=14, entry_tid=50828], [object_number=211, tag_tid=14, entry_tid=50827], [object_number=210, tag_tid=14, entry_tid=50826]]], state=connected] 2026-02-20T21:12:34.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:34.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:34.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:34.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=50829], [object_number=208, tag_tid=14, entry_tid=50828], [object_number=211, tag_tid=14, entry_tid=50827], [object_number=210, tag_tid=14, entry_tid=50826]]], state=connected] 2026-02-20T21:12:34.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=209, tag_tid=14, entry_tid=50829' 2026-02-20T21:12:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=209, tag_tid=14, entry_tid=50829' 2026-02-20T21:12:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=50829' '!=' 'object_number=210, tag_tid=14, entry_tid=50530' 2026-02-20T21:12:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=50829' = 'object_number=210, tag_tid=14, entry_tid=50530' 2026-02-20T21:12:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=209, tag_tid=14, entry_tid=50829' 2026-02-20T21:12:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:34.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:34.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:34.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=50829' 2026-02-20T21:12:34.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:34.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:34.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:34.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:34.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:34.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:34.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:34.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=50829' 2026-02-20T21:12:34.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:34.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:34.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50829 2026-02-20T21:12:34.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 50829 2026-02-20T21:12:34.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:34.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:34.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:34.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:34.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:34.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:34.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:34.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:34.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:34.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:34.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:34.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:34.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:34.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:34.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:34.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:34.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:34.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:34.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:34.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:34.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=51126], [object_number=209, tag_tid=14, entry_tid=51125], [object_number=208, tag_tid=14, entry_tid=51124], [object_number=211, tag_tid=14, entry_tid=51123]]], state=connected] 2026-02-20T21:12:34.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:34.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:34.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=51126], [object_number=209, tag_tid=14, entry_tid=51125], [object_number=208, tag_tid=14, entry_tid=51124], [object_number=211, tag_tid=14, entry_tid=51123]]], state=connected] 2026-02-20T21:12:34.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=210, tag_tid=14, entry_tid=51126' 2026-02-20T21:12:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=210, tag_tid=14, entry_tid=51126' 2026-02-20T21:12:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=51126' '!=' 'object_number=209, tag_tid=14, entry_tid=50829' 2026-02-20T21:12:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=51126' = 'object_number=209, tag_tid=14, entry_tid=50829' 2026-02-20T21:12:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=210, tag_tid=14, entry_tid=51126' 2026-02-20T21:12:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=51126' 2026-02-20T21:12:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:34.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:34.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:34.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=51126' 2026-02-20T21:12:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51126 2026-02-20T21:12:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 51126 2026-02-20T21:12:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:34.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:34.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:34.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:34.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:34.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:34.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:34.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:34.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:34.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:34.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:34.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:34.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:34.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:34.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:34.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:34.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:34.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:34.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:34.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:34.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:34.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:34.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:34.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:34.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:34.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:34.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:34.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:34.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:34.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:34.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:34.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:34.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:34.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:34.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:34.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:34.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=51424], [object_number=211, tag_tid=14, entry_tid=51423], [object_number=210, tag_tid=14, entry_tid=51422], [object_number=209, tag_tid=14, entry_tid=51421]]], state=connected] 2026-02-20T21:12:34.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:34.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:34.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:34.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:34.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:34.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:34.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:34.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:34.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:34.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:34.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:34.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:34.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:34.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:34.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:34.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:34.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:34.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:34.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=51424], [object_number=211, tag_tid=14, entry_tid=51423], [object_number=210, tag_tid=14, entry_tid=51422], [object_number=209, tag_tid=14, entry_tid=51421]]], state=connected] 2026-02-20T21:12:34.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=208, tag_tid=14, entry_tid=51424' 2026-02-20T21:12:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=208, tag_tid=14, entry_tid=51424' 2026-02-20T21:12:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=51424' '!=' 'object_number=210, tag_tid=14, entry_tid=51126' 2026-02-20T21:12:34.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=51424' = 'object_number=210, tag_tid=14, entry_tid=51126' 2026-02-20T21:12:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=208, tag_tid=14, entry_tid=51424' 2026-02-20T21:12:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:34.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=51424' 2026-02-20T21:12:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:34.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:34.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:34.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:34.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:34.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:34.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:34.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=51424' 2026-02-20T21:12:34.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:34.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51424 2026-02-20T21:12:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 51424 2026-02-20T21:12:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:34.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:34.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:34.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:34.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:34.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:34.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:34.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:34.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:34.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:34.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:34.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:34.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:34.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:34.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:34.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:34.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:34.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:34.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:35.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:35.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:35.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:35.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=51744], [object_number=211, tag_tid=14, entry_tid=51743], [object_number=210, tag_tid=14, entry_tid=51742], [object_number=209, tag_tid=14, entry_tid=51741]]], state=connected] 2026-02-20T21:12:35.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:35.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:35.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:35.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:35.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=51744], [object_number=211, tag_tid=14, entry_tid=51743], [object_number=210, tag_tid=14, entry_tid=51742], [object_number=209, tag_tid=14, entry_tid=51741]]], state=connected] 2026-02-20T21:12:35.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=208, tag_tid=14, entry_tid=51744' 2026-02-20T21:12:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=208, tag_tid=14, entry_tid=51744' 2026-02-20T21:12:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=51744' '!=' 'object_number=208, tag_tid=14, entry_tid=51424' 2026-02-20T21:12:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=51744' = 'object_number=208, tag_tid=14, entry_tid=51424' 2026-02-20T21:12:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=208, tag_tid=14, entry_tid=51744' 2026-02-20T21:12:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:35.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=51744' 2026-02-20T21:12:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:35.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=51744' 2026-02-20T21:12:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51744 2026-02-20T21:12:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 51744 2026-02-20T21:12:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:35.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:35.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:35.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:35.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:35.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:35.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:35.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:35.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:35.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:35.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:35.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:35.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:35.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:35.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:35.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:35.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:35.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:35.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:35.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:35.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:35.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:35.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:35.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:35.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:35.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:35.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:35.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:35.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:35.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:35.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:35.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:35.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:35.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:35.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:35.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:35.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:35.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:35.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:35.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:35.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:35.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:35.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=52076], [object_number=211, tag_tid=14, entry_tid=52075], [object_number=210, tag_tid=14, entry_tid=52074], [object_number=209, tag_tid=14, entry_tid=52073]]], state=connected] 2026-02-20T21:12:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:35.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:35.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:35.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:35.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:35.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:35.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=52076], [object_number=211, tag_tid=14, entry_tid=52075], [object_number=210, tag_tid=14, entry_tid=52074], [object_number=209, tag_tid=14, entry_tid=52073]]], state=connected] 2026-02-20T21:12:35.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=208, tag_tid=14, entry_tid=52076' 2026-02-20T21:12:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=208, tag_tid=14, entry_tid=52076' 2026-02-20T21:12:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=52076' '!=' 'object_number=208, tag_tid=14, entry_tid=51744' 2026-02-20T21:12:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=52076' = 'object_number=208, tag_tid=14, entry_tid=51744' 2026-02-20T21:12:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=208, tag_tid=14, entry_tid=52076' 2026-02-20T21:12:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=52076' 2026-02-20T21:12:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:35.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=52076' 2026-02-20T21:12:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52076 2026-02-20T21:12:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 52076 2026-02-20T21:12:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:35.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:35.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:35.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:35.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:35.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:35.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:35.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:35.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:35.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:35.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:35.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:35.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:35.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:35.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:35.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=52371], [object_number=210, tag_tid=14, entry_tid=52370], [object_number=209, tag_tid=14, entry_tid=52369], [object_number=208, tag_tid=14, entry_tid=52368]]], state=connected] 2026-02-20T21:12:35.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:35.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:35.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:35.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:35.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:35.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:35.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:35.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:35.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:35.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:35.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:35.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:35.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:35.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=52371], [object_number=210, tag_tid=14, entry_tid=52370], [object_number=209, tag_tid=14, entry_tid=52369], [object_number=208, tag_tid=14, entry_tid=52368]]], state=connected] 2026-02-20T21:12:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:35.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=211, tag_tid=14, entry_tid=52371' 2026-02-20T21:12:35.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=211, tag_tid=14, entry_tid=52371' 2026-02-20T21:12:35.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=52371' '!=' 'object_number=208, tag_tid=14, entry_tid=52076' 2026-02-20T21:12:35.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:35.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=52371' = 'object_number=208, tag_tid=14, entry_tid=52076' 2026-02-20T21:12:35.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=211, tag_tid=14, entry_tid=52371' 2026-02-20T21:12:35.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:35.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:35.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:35.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:35.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=52371' 2026-02-20T21:12:35.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:35.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=52371' 2026-02-20T21:12:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:35.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52371 2026-02-20T21:12:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 52371 2026-02-20T21:12:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:35.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:35.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:35.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:35.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:35.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:35.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:35.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:35.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:36.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:36.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:36.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=52638], [object_number=209, tag_tid=14, entry_tid=52637], [object_number=208, tag_tid=14, entry_tid=52636], [object_number=211, tag_tid=14, entry_tid=52635]]], state=connected] 2026-02-20T21:12:36.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=52638], [object_number=209, tag_tid=14, entry_tid=52637], [object_number=208, tag_tid=14, entry_tid=52636], [object_number=211, tag_tid=14, entry_tid=52635]]], state=connected] 2026-02-20T21:12:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=210, tag_tid=14, entry_tid=52638' 2026-02-20T21:12:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=210, tag_tid=14, entry_tid=52638' 2026-02-20T21:12:36.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=52638' '!=' 'object_number=211, tag_tid=14, entry_tid=52371' 2026-02-20T21:12:36.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:36.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=52638' = 'object_number=211, tag_tid=14, entry_tid=52371' 2026-02-20T21:12:36.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=210, tag_tid=14, entry_tid=52638' 2026-02-20T21:12:36.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:36.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:36.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:36.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:36.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=52638' 2026-02-20T21:12:36.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:36.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:36.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:36.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=52638' 2026-02-20T21:12:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52638 2026-02-20T21:12:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 52638 2026-02-20T21:12:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:36.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:36.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:36.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:36.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:36.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:36.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:36.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:36.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=52952], [object_number=211, tag_tid=14, entry_tid=52951], [object_number=210, tag_tid=14, entry_tid=52950], [object_number=209, tag_tid=14, entry_tid=52949]]], state=connected] 2026-02-20T21:12:36.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:36.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:36.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:36.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:36.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:36.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=52952], [object_number=211, tag_tid=14, entry_tid=52951], [object_number=210, tag_tid=14, entry_tid=52950], [object_number=209, tag_tid=14, entry_tid=52949]]], state=connected] 2026-02-20T21:12:36.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=208, tag_tid=14, entry_tid=52952' 2026-02-20T21:12:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=208, tag_tid=14, entry_tid=52952' 2026-02-20T21:12:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=52952' '!=' 'object_number=210, tag_tid=14, entry_tid=52638' 2026-02-20T21:12:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=52952' = 'object_number=210, tag_tid=14, entry_tid=52638' 2026-02-20T21:12:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=208, tag_tid=14, entry_tid=52952' 2026-02-20T21:12:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=52952' 2026-02-20T21:12:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:36.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:36.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=52952' 2026-02-20T21:12:36.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:36.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:36.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52952 2026-02-20T21:12:36.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 52952 2026-02-20T21:12:36.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:36.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:36.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:36.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:36.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:36.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:36.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:36.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:36.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:36.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:36.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:36.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:36.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:36.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:36.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:36.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:36.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:36.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:36.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:36.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=53286], [object_number=209, tag_tid=14, entry_tid=53285], [object_number=208, tag_tid=14, entry_tid=53284], [object_number=211, tag_tid=14, entry_tid=53283]]], state=connected] 2026-02-20T21:12:36.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:36.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:36.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:36.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:36.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:36.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:36.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:36.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:36.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:36.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:36.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:36.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:36.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:36.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:36.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:36.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:36.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:36.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=210, tag_tid=14, entry_tid=53286], [object_number=209, tag_tid=14, entry_tid=53285], [object_number=208, tag_tid=14, entry_tid=53284], [object_number=211, tag_tid=14, entry_tid=53283]]], state=connected] 2026-02-20T21:12:36.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=210, tag_tid=14, entry_tid=53286' 2026-02-20T21:12:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=210, tag_tid=14, entry_tid=53286' 2026-02-20T21:12:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=53286' '!=' 'object_number=208, tag_tid=14, entry_tid=52952' 2026-02-20T21:12:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=210, tag_tid=14, entry_tid=53286' = 'object_number=208, tag_tid=14, entry_tid=52952' 2026-02-20T21:12:36.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=210, tag_tid=14, entry_tid=53286' 2026-02-20T21:12:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=53286' 2026-02-20T21:12:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:36.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=210, tag_tid=14, entry_tid=53286' 2026-02-20T21:12:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53286 2026-02-20T21:12:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 53286 2026-02-20T21:12:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:36.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:36.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:36.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:36.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:36.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:37.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:37.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:37.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:37.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:37.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:37.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:37.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=53565], [object_number=208, tag_tid=14, entry_tid=53564], [object_number=211, tag_tid=14, entry_tid=53563], [object_number=210, tag_tid=14, entry_tid=53562]]], state=connected] 2026-02-20T21:12:37.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:37.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:37.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:37.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:37.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=209, tag_tid=14, entry_tid=53565], [object_number=208, tag_tid=14, entry_tid=53564], [object_number=211, tag_tid=14, entry_tid=53563], [object_number=210, tag_tid=14, entry_tid=53562]]], state=connected] 2026-02-20T21:12:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=209, tag_tid=14, entry_tid=53565' 2026-02-20T21:12:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=209, tag_tid=14, entry_tid=53565' 2026-02-20T21:12:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=53565' '!=' 'object_number=210, tag_tid=14, entry_tid=53286' 2026-02-20T21:12:37.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=209, tag_tid=14, entry_tid=53565' = 'object_number=210, tag_tid=14, entry_tid=53286' 2026-02-20T21:12:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=209, tag_tid=14, entry_tid=53565' 2026-02-20T21:12:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:37.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=53565' 2026-02-20T21:12:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:37.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:37.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:37.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:37.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:37.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:37.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:37.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=209, tag_tid=14, entry_tid=53565' 2026-02-20T21:12:37.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:37.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:37.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53565 2026-02-20T21:12:37.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 53565 2026-02-20T21:12:37.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:37.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:37.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:37.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:37.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:37.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:37.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:37.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:37.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:37.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:37.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:37.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:37.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:37.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:37.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:37.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:37.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:37.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=53871], [object_number=210, tag_tid=14, entry_tid=53870], [object_number=209, tag_tid=14, entry_tid=53869], [object_number=208, tag_tid=14, entry_tid=53868]]], state=connected] 2026-02-20T21:12:37.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:37.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:37.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=211, tag_tid=14, entry_tid=53871], [object_number=210, tag_tid=14, entry_tid=53870], [object_number=209, tag_tid=14, entry_tid=53869], [object_number=208, tag_tid=14, entry_tid=53868]]], state=connected] 2026-02-20T21:12:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=211, tag_tid=14, entry_tid=53871' 2026-02-20T21:12:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=211, tag_tid=14, entry_tid=53871' 2026-02-20T21:12:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=53871' '!=' 'object_number=209, tag_tid=14, entry_tid=53565' 2026-02-20T21:12:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=211, tag_tid=14, entry_tid=53871' = 'object_number=209, tag_tid=14, entry_tid=53565' 2026-02-20T21:12:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=211, tag_tid=14, entry_tid=53871' 2026-02-20T21:12:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:37.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=53871' 2026-02-20T21:12:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:37.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=211, tag_tid=14, entry_tid=53871' 2026-02-20T21:12:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53871 2026-02-20T21:12:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 53871 2026-02-20T21:12:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:37.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:37.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:37.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:37.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:37.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:37.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:37.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:37.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:37.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:37.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:37.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=54180], [object_number=211, tag_tid=14, entry_tid=54179], [object_number=210, tag_tid=14, entry_tid=54178], [object_number=209, tag_tid=14, entry_tid=54177]]], state=connected] 2026-02-20T21:12:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=54180], [object_number=211, tag_tid=14, entry_tid=54179], [object_number=210, tag_tid=14, entry_tid=54178], [object_number=209, tag_tid=14, entry_tid=54177]]], state=connected] 2026-02-20T21:12:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=208, tag_tid=14, entry_tid=54180' 2026-02-20T21:12:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=208, tag_tid=14, entry_tid=54180' 2026-02-20T21:12:37.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=54180' '!=' 'object_number=211, tag_tid=14, entry_tid=53871' 2026-02-20T21:12:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=54180' = 'object_number=211, tag_tid=14, entry_tid=53871' 2026-02-20T21:12:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=208, tag_tid=14, entry_tid=54180' 2026-02-20T21:12:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:37.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=54180' 2026-02-20T21:12:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:37.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=54180' 2026-02-20T21:12:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54180 2026-02-20T21:12:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 54180 2026-02-20T21:12:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:37.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:37.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:38.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:38.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:38.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:38.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=54488], [object_number=211, tag_tid=14, entry_tid=54487], [object_number=210, tag_tid=14, entry_tid=54486], [object_number=209, tag_tid=14, entry_tid=54485]]], state=connected] 2026-02-20T21:12:38.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:38.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:38.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 52 2026-02-20T21:12:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:38.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=208, tag_tid=14, entry_tid=54488], [object_number=211, tag_tid=14, entry_tid=54487], [object_number=210, tag_tid=14, entry_tid=54486], [object_number=209, tag_tid=14, entry_tid=54485]]], state=connected] 2026-02-20T21:12:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=208, tag_tid=14, entry_tid=54488' 2026-02-20T21:12:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=208, tag_tid=14, entry_tid=54488' 2026-02-20T21:12:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=54488' '!=' 'object_number=208, tag_tid=14, entry_tid=54180' 2026-02-20T21:12:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=208, tag_tid=14, entry_tid=54488' = 'object_number=208, tag_tid=14, entry_tid=54180' 2026-02-20T21:12:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=208, tag_tid=14, entry_tid=54488' 2026-02-20T21:12:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=54488' 2026-02-20T21:12:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:38.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:38.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:38.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=208, tag_tid=14, entry_tid=54488' 2026-02-20T21:12:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:38.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:38.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54488 2026-02-20T21:12:38.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 54488 2026-02-20T21:12:38.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:38.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:38.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:38.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:38.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:38.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:38.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:38.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:38.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:38.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:38.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:38.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:38.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:38.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:38.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:38.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=54739], [object_number=214, tag_tid=14, entry_tid=54738], [object_number=213, tag_tid=14, entry_tid=54737], [object_number=212, tag_tid=14, entry_tid=54736]]], state=connected] 2026-02-20T21:12:38.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:38.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:38.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:38.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:38.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:38.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:38.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=54739], [object_number=214, tag_tid=14, entry_tid=54738], [object_number=213, tag_tid=14, entry_tid=54737], [object_number=212, tag_tid=14, entry_tid=54736]]], state=connected] 2026-02-20T21:12:38.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=215, tag_tid=14, entry_tid=54739' 2026-02-20T21:12:38.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=215, tag_tid=14, entry_tid=54739' 2026-02-20T21:12:38.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=54739' '!=' 'object_number=208, tag_tid=14, entry_tid=54488' 2026-02-20T21:12:38.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:38.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=54739' = 'object_number=208, tag_tid=14, entry_tid=54488' 2026-02-20T21:12:38.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=215, tag_tid=14, entry_tid=54739' 2026-02-20T21:12:38.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:38.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:38.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:38.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:38.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=54739' 2026-02-20T21:12:38.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:38.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:38.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:38.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:38.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:38.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:38.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:38.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=54739' 2026-02-20T21:12:38.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:38.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:38.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54739 2026-02-20T21:12:38.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 54739 2026-02-20T21:12:38.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:38.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:38.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:38.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:38.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:38.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:38.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:38.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:38.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:38.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:38.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:38.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:38.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=55062], [object_number=213, tag_tid=14, entry_tid=55061], [object_number=212, tag_tid=14, entry_tid=55060], [object_number=215, tag_tid=14, entry_tid=55059]]], state=connected] 2026-02-20T21:12:38.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:38.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=55062], [object_number=213, tag_tid=14, entry_tid=55061], [object_number=212, tag_tid=14, entry_tid=55060], [object_number=215, tag_tid=14, entry_tid=55059]]], state=connected] 2026-02-20T21:12:38.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=214, tag_tid=14, entry_tid=55062' 2026-02-20T21:12:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=214, tag_tid=14, entry_tid=55062' 2026-02-20T21:12:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=55062' '!=' 'object_number=215, tag_tid=14, entry_tid=54739' 2026-02-20T21:12:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=55062' = 'object_number=215, tag_tid=14, entry_tid=54739' 2026-02-20T21:12:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=214, tag_tid=14, entry_tid=55062' 2026-02-20T21:12:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=55062' 2026-02-20T21:12:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:38.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:38.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=55062' 2026-02-20T21:12:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:38.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55062 2026-02-20T21:12:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 55062 2026-02-20T21:12:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:38.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:39.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:39.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:39.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:39.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:39.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:39.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:39.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:39.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:39.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:39.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:39.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:39.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=55402], [object_number=213, tag_tid=14, entry_tid=55401], [object_number=212, tag_tid=14, entry_tid=55400], [object_number=215, tag_tid=14, entry_tid=55399]]], state=connected] 2026-02-20T21:12:39.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:39.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:39.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:39.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:39.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=55402], [object_number=213, tag_tid=14, entry_tid=55401], [object_number=212, tag_tid=14, entry_tid=55400], [object_number=215, tag_tid=14, entry_tid=55399]]], state=connected] 2026-02-20T21:12:39.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=214, tag_tid=14, entry_tid=55402' 2026-02-20T21:12:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=214, tag_tid=14, entry_tid=55402' 2026-02-20T21:12:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=55402' '!=' 'object_number=214, tag_tid=14, entry_tid=55062' 2026-02-20T21:12:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=55402' = 'object_number=214, tag_tid=14, entry_tid=55062' 2026-02-20T21:12:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=214, tag_tid=14, entry_tid=55402' 2026-02-20T21:12:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=55402' 2026-02-20T21:12:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:39.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:39.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=55402' 2026-02-20T21:12:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:39.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:39.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55402 2026-02-20T21:12:39.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 55402 2026-02-20T21:12:39.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:39.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:39.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:39.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:39.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:39.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:39.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:39.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:39.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:39.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:39.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:39.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:39.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=55750], [object_number=213, tag_tid=14, entry_tid=55749], [object_number=212, tag_tid=14, entry_tid=55748], [object_number=215, tag_tid=14, entry_tid=55747]]], state=connected] 2026-02-20T21:12:39.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:39.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:39.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:39.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:39.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:39.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:39.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:39.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=55750], [object_number=213, tag_tid=14, entry_tid=55749], [object_number=212, tag_tid=14, entry_tid=55748], [object_number=215, tag_tid=14, entry_tid=55747]]], state=connected] 2026-02-20T21:12:39.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=214, tag_tid=14, entry_tid=55750' 2026-02-20T21:12:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=214, tag_tid=14, entry_tid=55750' 2026-02-20T21:12:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=55750' '!=' 'object_number=214, tag_tid=14, entry_tid=55402' 2026-02-20T21:12:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=55750' = 'object_number=214, tag_tid=14, entry_tid=55402' 2026-02-20T21:12:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=214, tag_tid=14, entry_tid=55750' 2026-02-20T21:12:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=55750' 2026-02-20T21:12:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:39.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:39.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:39.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:39.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:39.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:39.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=55750' 2026-02-20T21:12:39.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:39.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55750 2026-02-20T21:12:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 55750 2026-02-20T21:12:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:39.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:39.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:39.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:39.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:39.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:39.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:39.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:39.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:39.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:39.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:39.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:39.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:39.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:39.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:39.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:39.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:39.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:39.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:39.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:39.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=56091], [object_number=214, tag_tid=14, entry_tid=56090], [object_number=213, tag_tid=14, entry_tid=56089], [object_number=212, tag_tid=14, entry_tid=56088]]], state=connected] 2026-02-20T21:12:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:39.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:39.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:39.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=56091], [object_number=214, tag_tid=14, entry_tid=56090], [object_number=213, tag_tid=14, entry_tid=56089], [object_number=212, tag_tid=14, entry_tid=56088]]], state=connected] 2026-02-20T21:12:39.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:39.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=215, tag_tid=14, entry_tid=56091' 2026-02-20T21:12:39.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=215, tag_tid=14, entry_tid=56091' 2026-02-20T21:12:39.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=56091' '!=' 'object_number=214, tag_tid=14, entry_tid=55750' 2026-02-20T21:12:39.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:39.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=56091' = 'object_number=214, tag_tid=14, entry_tid=55750' 2026-02-20T21:12:39.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=215, tag_tid=14, entry_tid=56091' 2026-02-20T21:12:39.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:39.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:39.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:39.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:39.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=56091' 2026-02-20T21:12:39.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:39.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:39.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:39.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:39.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:39.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:39.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:39.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=56091' 2026-02-20T21:12:39.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:39.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:39.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56091 2026-02-20T21:12:39.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 56091 2026-02-20T21:12:39.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:39.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:39.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:40.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:40.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:40.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:40.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:40.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=56438], [object_number=213, tag_tid=14, entry_tid=56437], [object_number=212, tag_tid=14, entry_tid=56436], [object_number=215, tag_tid=14, entry_tid=56435]]], state=connected] 2026-02-20T21:12:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:40.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:40.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:40.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:40.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:40.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:40.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=214, tag_tid=14, entry_tid=56438], [object_number=213, tag_tid=14, entry_tid=56437], [object_number=212, tag_tid=14, entry_tid=56436], [object_number=215, tag_tid=14, entry_tid=56435]]], state=connected] 2026-02-20T21:12:40.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=214, tag_tid=14, entry_tid=56438' 2026-02-20T21:12:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=214, tag_tid=14, entry_tid=56438' 2026-02-20T21:12:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=56438' '!=' 'object_number=215, tag_tid=14, entry_tid=56091' 2026-02-20T21:12:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=214, tag_tid=14, entry_tid=56438' = 'object_number=215, tag_tid=14, entry_tid=56091' 2026-02-20T21:12:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=214, tag_tid=14, entry_tid=56438' 2026-02-20T21:12:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:40.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:40.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:40.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=56438' 2026-02-20T21:12:40.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:40.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=214, tag_tid=14, entry_tid=56438' 2026-02-20T21:12:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56438 2026-02-20T21:12:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 56438 2026-02-20T21:12:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:40.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:40.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:40.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:40.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:40.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:40.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:40.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:40.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:40.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:40.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:40.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:40.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:40.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:40.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:40.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:40.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=56784], [object_number=215, tag_tid=14, entry_tid=56783], [object_number=214, tag_tid=14, entry_tid=56782], [object_number=213, tag_tid=14, entry_tid=56781]]], state=connected] 2026-02-20T21:12:40.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:40.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:40.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:40.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:40.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:40.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:40.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:40.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:40.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=56784], [object_number=215, tag_tid=14, entry_tid=56783], [object_number=214, tag_tid=14, entry_tid=56782], [object_number=213, tag_tid=14, entry_tid=56781]]], state=connected] 2026-02-20T21:12:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=212, tag_tid=14, entry_tid=56784' 2026-02-20T21:12:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=212, tag_tid=14, entry_tid=56784' 2026-02-20T21:12:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=56784' '!=' 'object_number=214, tag_tid=14, entry_tid=56438' 2026-02-20T21:12:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=56784' = 'object_number=214, tag_tid=14, entry_tid=56438' 2026-02-20T21:12:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=212, tag_tid=14, entry_tid=56784' 2026-02-20T21:12:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=56784' 2026-02-20T21:12:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=56784' 2026-02-20T21:12:40.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:40.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:40.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56784 2026-02-20T21:12:40.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 56784 2026-02-20T21:12:40.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:40.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:40.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:40.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:40.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:40.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:40.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:40.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:40.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:40.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:40.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:40.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:40.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:40.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:40.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=57135], [object_number=214, tag_tid=14, entry_tid=57134], [object_number=213, tag_tid=14, entry_tid=57133], [object_number=212, tag_tid=14, entry_tid=57132]]], state=connected] 2026-02-20T21:12:40.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:40.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=57135], [object_number=214, tag_tid=14, entry_tid=57134], [object_number=213, tag_tid=14, entry_tid=57133], [object_number=212, tag_tid=14, entry_tid=57132]]], state=connected] 2026-02-20T21:12:40.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=215, tag_tid=14, entry_tid=57135' 2026-02-20T21:12:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=215, tag_tid=14, entry_tid=57135' 2026-02-20T21:12:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=57135' '!=' 'object_number=212, tag_tid=14, entry_tid=56784' 2026-02-20T21:12:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=57135' = 'object_number=212, tag_tid=14, entry_tid=56784' 2026-02-20T21:12:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=215, tag_tid=14, entry_tid=57135' 2026-02-20T21:12:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:40.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:40.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=57135' 2026-02-20T21:12:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:40.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:40.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=57135' 2026-02-20T21:12:40.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:40.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:40.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57135 2026-02-20T21:12:40.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 57135 2026-02-20T21:12:40.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:40.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:40.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:41.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:41.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:41.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:41.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:41.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:41.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:41.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:41.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:41.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:41.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:41.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:41.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:41.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:41.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:41.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:41.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:41.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:41.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:41.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:41.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:41.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:41.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:41.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:41.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:41.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:41.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:41.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:41.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:41.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:41.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:41.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:41.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:41.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:41.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:41.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:41.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:41.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:41.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:41.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:41.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:41.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:41.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:41.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:41.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:41.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:41.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:41.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=57483], [object_number=214, tag_tid=14, entry_tid=57482], [object_number=213, tag_tid=14, entry_tid=57481], [object_number=212, tag_tid=14, entry_tid=57480]]], state=connected] 2026-02-20T21:12:41.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:41.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:41.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:41.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:41.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:41.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:41.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:41.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:41.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:41.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:41.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:41.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:41.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:41.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:41.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:41.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:41.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=215, tag_tid=14, entry_tid=57483], [object_number=214, tag_tid=14, entry_tid=57482], [object_number=213, tag_tid=14, entry_tid=57481], [object_number=212, tag_tid=14, entry_tid=57480]]], state=connected] 2026-02-20T21:12:41.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=215, tag_tid=14, entry_tid=57483' 2026-02-20T21:12:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=215, tag_tid=14, entry_tid=57483' 2026-02-20T21:12:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=57483' '!=' 'object_number=215, tag_tid=14, entry_tid=57135' 2026-02-20T21:12:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=215, tag_tid=14, entry_tid=57483' = 'object_number=215, tag_tid=14, entry_tid=57135' 2026-02-20T21:12:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=215, tag_tid=14, entry_tid=57483' 2026-02-20T21:12:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:41.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=57483' 2026-02-20T21:12:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:41.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:41.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:41.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:41.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=215, tag_tid=14, entry_tid=57483' 2026-02-20T21:12:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:41.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57483 2026-02-20T21:12:41.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 57483 2026-02-20T21:12:41.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:41.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:41.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:41.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:41.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:41.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:41.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:41.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:41.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:41.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=57805], [object_number=212, tag_tid=14, entry_tid=57804], [object_number=215, tag_tid=14, entry_tid=57803], [object_number=214, tag_tid=14, entry_tid=57802]]], state=connected] 2026-02-20T21:12:41.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:41.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:41.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:41.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=57805], [object_number=212, tag_tid=14, entry_tid=57804], [object_number=215, tag_tid=14, entry_tid=57803], [object_number=214, tag_tid=14, entry_tid=57802]]], state=connected] 2026-02-20T21:12:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=213, tag_tid=14, entry_tid=57805' 2026-02-20T21:12:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=213, tag_tid=14, entry_tid=57805' 2026-02-20T21:12:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=57805' '!=' 'object_number=215, tag_tid=14, entry_tid=57483' 2026-02-20T21:12:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=57805' = 'object_number=215, tag_tid=14, entry_tid=57483' 2026-02-20T21:12:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=213, tag_tid=14, entry_tid=57805' 2026-02-20T21:12:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=57805' 2026-02-20T21:12:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=57805' 2026-02-20T21:12:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:41.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57805 2026-02-20T21:12:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 57805 2026-02-20T21:12:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:41.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:41.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:41.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:41.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:41.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:42.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:42.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:42.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:42.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:42.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:42.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:42.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:42.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:42.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:42.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=58325], [object_number=212, tag_tid=14, entry_tid=58324], [object_number=215, tag_tid=14, entry_tid=58323], [object_number=214, tag_tid=14, entry_tid=58322]]], state=connected] 2026-02-20T21:12:42.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:42.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:42.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:42.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:42.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=213, tag_tid=14, entry_tid=58325], [object_number=212, tag_tid=14, entry_tid=58324], [object_number=215, tag_tid=14, entry_tid=58323], [object_number=214, tag_tid=14, entry_tid=58322]]], state=connected] 2026-02-20T21:12:42.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=213, tag_tid=14, entry_tid=58325' 2026-02-20T21:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=213, tag_tid=14, entry_tid=58325' 2026-02-20T21:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=58325' '!=' 'object_number=213, tag_tid=14, entry_tid=57805' 2026-02-20T21:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=213, tag_tid=14, entry_tid=58325' = 'object_number=213, tag_tid=14, entry_tid=57805' 2026-02-20T21:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=213, tag_tid=14, entry_tid=58325' 2026-02-20T21:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:42.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=58325' 2026-02-20T21:12:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:42.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=213, tag_tid=14, entry_tid=58325' 2026-02-20T21:12:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58325 2026-02-20T21:12:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 58325 2026-02-20T21:12:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:42.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:42.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:42.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:42.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:42.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=58648], [object_number=215, tag_tid=14, entry_tid=58647], [object_number=214, tag_tid=14, entry_tid=58646], [object_number=213, tag_tid=14, entry_tid=58645]]], state=connected] 2026-02-20T21:12:42.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:42.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:42.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:42.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 53 2026-02-20T21:12:42.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=212, tag_tid=14, entry_tid=58648], [object_number=215, tag_tid=14, entry_tid=58647], [object_number=214, tag_tid=14, entry_tid=58646], [object_number=213, tag_tid=14, entry_tid=58645]]], state=connected] 2026-02-20T21:12:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=212, tag_tid=14, entry_tid=58648' 2026-02-20T21:12:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=212, tag_tid=14, entry_tid=58648' 2026-02-20T21:12:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=58648' '!=' 'object_number=213, tag_tid=14, entry_tid=58325' 2026-02-20T21:12:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=212, tag_tid=14, entry_tid=58648' = 'object_number=213, tag_tid=14, entry_tid=58325' 2026-02-20T21:12:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=212, tag_tid=14, entry_tid=58648' 2026-02-20T21:12:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=58648' 2026-02-20T21:12:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:42.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=212, tag_tid=14, entry_tid=58648' 2026-02-20T21:12:42.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:42.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:42.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58648 2026-02-20T21:12:42.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 58648 2026-02-20T21:12:42.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:42.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:42.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:42.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:42.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:42.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:42.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:42.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:42.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:42.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:42.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:42.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=58889], [object_number=216, tag_tid=14, entry_tid=58888], [object_number=219, tag_tid=14, entry_tid=58887], [object_number=218, tag_tid=14, entry_tid=58886]]], state=connected] 2026-02-20T21:12:42.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:42.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:42.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:42.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=58889], [object_number=216, tag_tid=14, entry_tid=58888], [object_number=219, tag_tid=14, entry_tid=58887], [object_number=218, tag_tid=14, entry_tid=58886]]], state=connected] 2026-02-20T21:12:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:42.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=217, tag_tid=14, entry_tid=58889' 2026-02-20T21:12:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=14, entry_tid=58889' 2026-02-20T21:12:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=58889' '!=' 'object_number=212, tag_tid=14, entry_tid=58648' 2026-02-20T21:12:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=58889' = 'object_number=212, tag_tid=14, entry_tid=58648' 2026-02-20T21:12:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=217, tag_tid=14, entry_tid=58889' 2026-02-20T21:12:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:42.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:42.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=58889' 2026-02-20T21:12:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:42.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:42.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:42.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:42.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=58889' 2026-02-20T21:12:42.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:42.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:42.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58889 2026-02-20T21:12:42.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 58889 2026-02-20T21:12:42.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:42.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:42.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:42.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:42.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:42.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:42.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:42.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:42.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:42.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:42.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:43.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:43.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:43.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:43.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:43.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:43.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:43.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:43.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:43.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:43.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:43.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:43.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:43.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:43.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:43.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:43.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:43.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:43.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:43.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=59184], [object_number=219, tag_tid=14, entry_tid=59183], [object_number=218, tag_tid=14, entry_tid=59182], [object_number=217, tag_tid=14, entry_tid=59181]]], state=connected] 2026-02-20T21:12:43.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:43.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:43.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=59184], [object_number=219, tag_tid=14, entry_tid=59183], [object_number=218, tag_tid=14, entry_tid=59182], [object_number=217, tag_tid=14, entry_tid=59181]]], state=connected] 2026-02-20T21:12:43.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=216, tag_tid=14, entry_tid=59184' 2026-02-20T21:12:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=216, tag_tid=14, entry_tid=59184' 2026-02-20T21:12:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=59184' '!=' 'object_number=217, tag_tid=14, entry_tid=58889' 2026-02-20T21:12:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=59184' = 'object_number=217, tag_tid=14, entry_tid=58889' 2026-02-20T21:12:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=216, tag_tid=14, entry_tid=59184' 2026-02-20T21:12:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:43.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=59184' 2026-02-20T21:12:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:43.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:43.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=59184' 2026-02-20T21:12:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:43.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=59184 2026-02-20T21:12:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 59184 2026-02-20T21:12:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:43.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:43.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:43.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:43.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:43.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:43.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:43.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:43.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:43.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:43.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:43.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:43.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:43.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:43.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=59502], [object_number=217, tag_tid=14, entry_tid=59501], [object_number=216, tag_tid=14, entry_tid=59500], [object_number=219, tag_tid=14, entry_tid=59499]]], state=connected] 2026-02-20T21:12:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:43.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:43.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:43.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:43.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=59502], [object_number=217, tag_tid=14, entry_tid=59501], [object_number=216, tag_tid=14, entry_tid=59500], [object_number=219, tag_tid=14, entry_tid=59499]]], state=connected] 2026-02-20T21:12:43.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=218, tag_tid=14, entry_tid=59502' 2026-02-20T21:12:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=218, tag_tid=14, entry_tid=59502' 2026-02-20T21:12:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=59502' '!=' 'object_number=216, tag_tid=14, entry_tid=59184' 2026-02-20T21:12:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=59502' = 'object_number=216, tag_tid=14, entry_tid=59184' 2026-02-20T21:12:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=218, tag_tid=14, entry_tid=59502' 2026-02-20T21:12:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:43.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:43.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=59502' 2026-02-20T21:12:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:43.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:43.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=59502' 2026-02-20T21:12:43.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:43.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=59502 2026-02-20T21:12:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 59502 2026-02-20T21:12:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:43.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:43.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:43.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:43.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:43.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:43.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:43.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:43.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:43.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:43.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:43.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:43.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:43.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:43.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:43.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:43.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:43.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:43.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:43.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:43.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:43.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:43.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:43.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:43.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:43.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:43.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:43.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:43.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:43.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:43.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:43.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:43.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:43.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:43.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:43.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:43.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:43.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:43.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:43.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:43.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:43.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=59835], [object_number=218, tag_tid=14, entry_tid=59834], [object_number=217, tag_tid=14, entry_tid=59833], [object_number=216, tag_tid=14, entry_tid=59832]]], state=connected] 2026-02-20T21:12:43.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:43.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:43.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:43.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:43.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:43.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=59835], [object_number=218, tag_tid=14, entry_tid=59834], [object_number=217, tag_tid=14, entry_tid=59833], [object_number=216, tag_tid=14, entry_tid=59832]]], state=connected] 2026-02-20T21:12:43.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=219, tag_tid=14, entry_tid=59835' 2026-02-20T21:12:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=219, tag_tid=14, entry_tid=59835' 2026-02-20T21:12:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=59835' '!=' 'object_number=218, tag_tid=14, entry_tid=59502' 2026-02-20T21:12:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=59835' = 'object_number=218, tag_tid=14, entry_tid=59502' 2026-02-20T21:12:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=219, tag_tid=14, entry_tid=59835' 2026-02-20T21:12:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:43.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:43.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:43.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=59835' 2026-02-20T21:12:43.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:43.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:43.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=59835' 2026-02-20T21:12:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:43.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=59835 2026-02-20T21:12:43.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 59835 2026-02-20T21:12:43.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:43.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:43.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:43.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:43.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:43.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:43.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:43.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:43.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:43.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:43.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:43.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:43.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:43.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:44.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:44.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=60135], [object_number=218, tag_tid=14, entry_tid=60134], [object_number=217, tag_tid=14, entry_tid=60133], [object_number=216, tag_tid=14, entry_tid=60132]]], state=connected] 2026-02-20T21:12:44.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:44.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:44.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:44.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:44.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=219, tag_tid=14, entry_tid=60135], [object_number=218, tag_tid=14, entry_tid=60134], [object_number=217, tag_tid=14, entry_tid=60133], [object_number=216, tag_tid=14, entry_tid=60132]]], state=connected] 2026-02-20T21:12:44.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=219, tag_tid=14, entry_tid=60135' 2026-02-20T21:12:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=219, tag_tid=14, entry_tid=60135' 2026-02-20T21:12:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=60135' '!=' 'object_number=219, tag_tid=14, entry_tid=59835' 2026-02-20T21:12:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=219, tag_tid=14, entry_tid=60135' = 'object_number=219, tag_tid=14, entry_tid=59835' 2026-02-20T21:12:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=219, tag_tid=14, entry_tid=60135' 2026-02-20T21:12:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:44.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:44.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:44.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=60135' 2026-02-20T21:12:44.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:44.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:44.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:44.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:44.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=219, tag_tid=14, entry_tid=60135' 2026-02-20T21:12:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:44.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=60135 2026-02-20T21:12:44.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 60135 2026-02-20T21:12:44.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:44.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:44.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:44.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:44.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:44.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:44.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:44.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:44.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:44.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:44.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:44.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:44.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:44.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=60442], [object_number=217, tag_tid=14, entry_tid=60441], [object_number=216, tag_tid=14, entry_tid=60440], [object_number=219, tag_tid=14, entry_tid=60439]]], state=connected] 2026-02-20T21:12:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:44.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:44.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=60442], [object_number=217, tag_tid=14, entry_tid=60441], [object_number=216, tag_tid=14, entry_tid=60440], [object_number=219, tag_tid=14, entry_tid=60439]]], state=connected] 2026-02-20T21:12:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=218, tag_tid=14, entry_tid=60442' 2026-02-20T21:12:44.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=218, tag_tid=14, entry_tid=60442' 2026-02-20T21:12:44.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=60442' '!=' 'object_number=219, tag_tid=14, entry_tid=60135' 2026-02-20T21:12:44.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:44.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=60442' = 'object_number=219, tag_tid=14, entry_tid=60135' 2026-02-20T21:12:44.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=218, tag_tid=14, entry_tid=60442' 2026-02-20T21:12:44.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=60442' 2026-02-20T21:12:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:44.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:44.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:44.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:44.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=60442' 2026-02-20T21:12:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:44.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:44.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=60442 2026-02-20T21:12:44.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 60442 2026-02-20T21:12:44.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:44.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:44.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:44.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:44.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:44.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:44.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:44.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:44.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:44.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:44.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:44.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:44.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:44.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:44.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:44.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:44.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=60744], [object_number=219, tag_tid=14, entry_tid=60743], [object_number=218, tag_tid=14, entry_tid=60742], [object_number=217, tag_tid=14, entry_tid=60741]]], state=connected] 2026-02-20T21:12:44.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:44.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:44.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:44.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:44.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:44.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:44.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:44.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:44.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=60744], [object_number=219, tag_tid=14, entry_tid=60743], [object_number=218, tag_tid=14, entry_tid=60742], [object_number=217, tag_tid=14, entry_tid=60741]]], state=connected] 2026-02-20T21:12:44.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=216, tag_tid=14, entry_tid=60744' 2026-02-20T21:12:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=216, tag_tid=14, entry_tid=60744' 2026-02-20T21:12:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=60744' '!=' 'object_number=218, tag_tid=14, entry_tid=60442' 2026-02-20T21:12:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=60744' = 'object_number=218, tag_tid=14, entry_tid=60442' 2026-02-20T21:12:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=216, tag_tid=14, entry_tid=60744' 2026-02-20T21:12:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=60744' 2026-02-20T21:12:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:44.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=60744' 2026-02-20T21:12:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=60744 2026-02-20T21:12:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 60744 2026-02-20T21:12:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:44.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:44.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:44.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:44.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:44.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:44.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:44.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:44.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:45.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:45.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:45.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:45.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:45.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:45.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:45.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:45.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:45.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:45.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=61057], [object_number=216, tag_tid=14, entry_tid=61056], [object_number=219, tag_tid=14, entry_tid=61055], [object_number=218, tag_tid=14, entry_tid=61054]]], state=connected] 2026-02-20T21:12:45.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:45.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:45.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:45.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:45.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:45.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:45.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=61057], [object_number=216, tag_tid=14, entry_tid=61056], [object_number=219, tag_tid=14, entry_tid=61055], [object_number=218, tag_tid=14, entry_tid=61054]]], state=connected] 2026-02-20T21:12:45.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=217, tag_tid=14, entry_tid=61057' 2026-02-20T21:12:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=14, entry_tid=61057' 2026-02-20T21:12:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=61057' '!=' 'object_number=216, tag_tid=14, entry_tid=60744' 2026-02-20T21:12:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=61057' = 'object_number=216, tag_tid=14, entry_tid=60744' 2026-02-20T21:12:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=217, tag_tid=14, entry_tid=61057' 2026-02-20T21:12:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:45.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=61057' 2026-02-20T21:12:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:45.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:45.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:45.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:45.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=61057' 2026-02-20T21:12:45.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=61057 2026-02-20T21:12:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 61057 2026-02-20T21:12:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:45.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:45.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:45.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:45.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:45.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:45.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:45.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:45.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:45.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:45.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:45.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:45.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:45.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:45.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:45.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=61388], [object_number=219, tag_tid=14, entry_tid=61387], [object_number=218, tag_tid=14, entry_tid=61386], [object_number=217, tag_tid=14, entry_tid=61385]]], state=connected] 2026-02-20T21:12:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:45.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:45.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=216, tag_tid=14, entry_tid=61388], [object_number=219, tag_tid=14, entry_tid=61387], [object_number=218, tag_tid=14, entry_tid=61386], [object_number=217, tag_tid=14, entry_tid=61385]]], state=connected] 2026-02-20T21:12:45.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=216, tag_tid=14, entry_tid=61388' 2026-02-20T21:12:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=216, tag_tid=14, entry_tid=61388' 2026-02-20T21:12:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=61388' '!=' 'object_number=217, tag_tid=14, entry_tid=61057' 2026-02-20T21:12:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=216, tag_tid=14, entry_tid=61388' = 'object_number=217, tag_tid=14, entry_tid=61057' 2026-02-20T21:12:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=216, tag_tid=14, entry_tid=61388' 2026-02-20T21:12:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:45.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=61388' 2026-02-20T21:12:45.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:45.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:45.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:45.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:45.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:45.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:45.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=216, tag_tid=14, entry_tid=61388' 2026-02-20T21:12:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:45.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=61388 2026-02-20T21:12:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 61388 2026-02-20T21:12:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:45.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:45.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:45.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:45.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:45.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:45.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:45.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:45.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:45.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:45.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:45.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:45.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:45.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:45.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:45.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:45.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:45.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:45.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:45.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:45.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:45.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:45.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:45.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:45.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:45.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:45.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:45.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:45.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:45.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=61722], [object_number=217, tag_tid=14, entry_tid=61721], [object_number=216, tag_tid=14, entry_tid=61720], [object_number=219, tag_tid=14, entry_tid=61719]]], state=connected] 2026-02-20T21:12:45.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:45.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:45.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:45.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:45.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:45.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:45.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:45.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:45.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:45.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:45.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:45.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=61722], [object_number=217, tag_tid=14, entry_tid=61721], [object_number=216, tag_tid=14, entry_tid=61720], [object_number=219, tag_tid=14, entry_tid=61719]]], state=connected] 2026-02-20T21:12:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=218, tag_tid=14, entry_tid=61722' 2026-02-20T21:12:45.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=218, tag_tid=14, entry_tid=61722' 2026-02-20T21:12:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=61722' '!=' 'object_number=216, tag_tid=14, entry_tid=61388' 2026-02-20T21:12:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=61722' = 'object_number=216, tag_tid=14, entry_tid=61388' 2026-02-20T21:12:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=218, tag_tid=14, entry_tid=61722' 2026-02-20T21:12:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=61722' 2026-02-20T21:12:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:45.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=61722' 2026-02-20T21:12:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=61722 2026-02-20T21:12:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 61722 2026-02-20T21:12:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:46.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:46.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:46.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:46.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:46.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:46.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:46.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:46.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:46.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:46.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:46.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:46.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:46.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:46.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:46.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:46.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:46.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:46.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:46.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:46.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:46.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:46.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:46.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:46.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:46.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:46.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:46.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:46.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=62050], [object_number=217, tag_tid=14, entry_tid=62049], [object_number=216, tag_tid=14, entry_tid=62048], [object_number=219, tag_tid=14, entry_tid=62047]]], state=connected] 2026-02-20T21:12:46.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:46.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:46.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:46.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:46.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=218, tag_tid=14, entry_tid=62050], [object_number=217, tag_tid=14, entry_tid=62049], [object_number=216, tag_tid=14, entry_tid=62048], [object_number=219, tag_tid=14, entry_tid=62047]]], state=connected] 2026-02-20T21:12:46.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=218, tag_tid=14, entry_tid=62050' 2026-02-20T21:12:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=218, tag_tid=14, entry_tid=62050' 2026-02-20T21:12:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=62050' '!=' 'object_number=218, tag_tid=14, entry_tid=61722' 2026-02-20T21:12:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=218, tag_tid=14, entry_tid=62050' = 'object_number=218, tag_tid=14, entry_tid=61722' 2026-02-20T21:12:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=218, tag_tid=14, entry_tid=62050' 2026-02-20T21:12:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:46.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:46.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:46.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=62050' 2026-02-20T21:12:46.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:46.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:46.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:46.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:46.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:46.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:46.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=218, tag_tid=14, entry_tid=62050' 2026-02-20T21:12:46.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:46.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:46.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=62050 2026-02-20T21:12:46.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 62050 2026-02-20T21:12:46.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:46.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:46.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:46.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:46.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:46.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:46.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:46.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:46.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:46.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:46.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:46.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:46.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:46.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:46.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:46.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:46.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:46.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:46.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:46.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:46.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:46.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:46.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:46.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:46.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:46.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:46.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:46.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:46.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:46.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:46.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:46.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:46.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=62101], [object_number=216, tag_tid=14, entry_tid=62100], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:46.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:46.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:46.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:46.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:46.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:46.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:46.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:46.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:46.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:46.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:46.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:46.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:46.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:46.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:46.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:46.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=62101], [object_number=216, tag_tid=14, entry_tid=62100], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=62101' '!=' 'object_number=218, tag_tid=14, entry_tid=62050' 2026-02-20T21:12:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:12:46.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=62101' = 'object_number=218, tag_tid=14, entry_tid=62050' 2026-02-20T21:12:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:46.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:46.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=15 2026-02-20T21:12:46.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:46.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:12:46.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:46.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=14 2026-02-20T21:12:46.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:46.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:46.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:46.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:12:46.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:46.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:12:46.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:12:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=62101 2026-02-20T21:12:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 15 = 14 -a 1 -le 62101 2026-02-20T21:12:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:12:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:12:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:46.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:46.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:46.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:46.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:46.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:46.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:46.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:46.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:46.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:46.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:46.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:46.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:46.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:46.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:46.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:46.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:46.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:46.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:46.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:46.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:46.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:46.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:46.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:46.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=62101], [object_number=216, tag_tid=14, entry_tid=62100], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:46.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:46.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:46.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:46.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:46.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:46.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:46.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:46.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:46.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:46.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:46.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:46.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:46.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=62101], [object_number=216, tag_tid=14, entry_tid=62100], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:46.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:46.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:46.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=62101' '!=' 'object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:46.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:46.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:12:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:47.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:47.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:47.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:47.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:47.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:47.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:47.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:47.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:47.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:47.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:47.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:47.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:47.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:47.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:47.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:47.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:47.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=62101], [object_number=216, tag_tid=14, entry_tid=62100], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:47.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:47.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:47.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:47.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:47.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=62101], [object_number=216, tag_tid=14, entry_tid=62100], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=62101' '!=' 'object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:12:48.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:48.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:48.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:48.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:48.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:48.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:48.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:48.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:48.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=62101], [object_number=216, tag_tid=14, entry_tid=62100], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:48.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=62101], [object_number=216, tag_tid=14, entry_tid=62100], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:48.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=62101' '!=' 'object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:48.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:12:49.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:49.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:49.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:49.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:49.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:49.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:49.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:49.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:49.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:49.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:49.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:49.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:49.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:49.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:49.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:49.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:49.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:49.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:49.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:49.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:49.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:49.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=62101], [object_number=216, tag_tid=14, entry_tid=62100], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:49.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:49.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:50.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=14, entry_tid=62101], [object_number=216, tag_tid=14, entry_tid=62100], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:50.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:50.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:50.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:50.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=217, tag_tid=14, entry_tid=62101' '!=' 'object_number=217, tag_tid=14, entry_tid=62101' 2026-02-20T21:12:50.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:12:50.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:12:52.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:12:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:12:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:12:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:12:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:12:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:12:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:52.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:12:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:12:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:12:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:12:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:12:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:12:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:12:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:12:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:12:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:12:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:12:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:12:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:12:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:12:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:12:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:12:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:12:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:12:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:12:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:52.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:12:52.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:12:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:12:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:12:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:12:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:12:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:12:52.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:12:52.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:12:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:12:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:12:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:52.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:52.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:52.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:52.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:52.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:52.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:12:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:12:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:12:52.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:12:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:12:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:12:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:12:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:12:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:12:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:12:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:52.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 54 2026-02-20T21:12:52.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 54 2026-02-20T21:12:52.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:12:52.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:52.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=217, tag_tid=15, entry_tid=1], [object_number=216, tag_tid=15, entry_tid=0], [object_number=219, tag_tid=14, entry_tid=62099], [object_number=218, tag_tid=14, entry_tid=62098]]], state=connected] 2026-02-20T21:12:52.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:12:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=217, tag_tid=15, entry_tid=1' -a 'object_number=217, tag_tid=15, entry_tid=1' = 'object_number=217, tag_tid=15, entry_tid=1' 2026-02-20T21:12:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap5 2026-02-20T21:12:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:12:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:52.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5 2026-02-20T21:12:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:12:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:12:52.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:12:53.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap5 2026-02-20T21:12:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-20T21:12:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:12:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:12:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 9 2026-02-20T21:12:53.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ba958b175d 2026-02-20T21:12:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ba958b175d 2026-02-20T21:12:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:12:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:12:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:12:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:12:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:06:40 2026 2026-02-20T21:12:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:11:34 2026 2026-02-20T21:12:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:12:29 2026 2026-02-20T21:12:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:12:53.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ba958b175d 2026-02-20T21:12:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:12:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:12:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:12:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:12:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap4-clean ']' 2026-02-20T21:12:53.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap4-clean 2026-02-20T21:12:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4-clean 2026-02-20T21:12:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:12:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:12:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:12:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:12:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:12:53.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:12:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:12:53.590 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.588+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:12:53.591 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.590+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.591 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.590+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.591 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.590+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.592 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.590+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.592 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.591+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.670 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.669+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:12:53.671 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.670+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:12:53.749 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.748+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:12:53.751 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.749+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.751 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.750+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.751 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.750+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.751 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.750+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.786 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.785+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:12:53.788 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.787+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.788 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.787+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.789 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.787+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.789 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:53.787+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:53.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:12:54.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:12:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:12:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap5 2026-02-20T21:12:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5 2026-02-20T21:12:54.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:12:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:12:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:12:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:12:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:12:54.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:12:54.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:12:54.740 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.738+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:12:54.741 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.740+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:54.741 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.740+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:54.742 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.740+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:54.742 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.741+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:54.821 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.820+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:12:54.821 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.820+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:12:54.821 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.820+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:12:54.856 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:12:54.855+0000 7f412e4cf640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:12:54.856 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:12:54.855+0000 7f4128cc4640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:12:54.896 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.895+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:54.897 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.895+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:54.897 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.895+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:54.897 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.895+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:54.897 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.895+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:54.897 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.895+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:12:54.932 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.931+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:12:54.932 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.931+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:12:54.932 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.931+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:12:54.933 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:54.931+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:12:55.004 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:55.002+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:12:55.004 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:55.002+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:12:55.004 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:55.002+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:12:55.004 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:55.002+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:12:55.004 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:55.002+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:12:55.040 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:55.039+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:12:55.042 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:55.040+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:12:55.042 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:55.040+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:12:55.042 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:55.040+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:12:55.042 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:12:55.040+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:12:55.112 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:12:55.110+0000 7f412c4cb640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:12:55.112 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:12:55.110+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:12:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:12:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:12:55.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:12:55.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:12:55.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap5-clean 2026-02-20T21:12:55.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap5-clean 2026-02-20T21:12:55.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:12:55.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:55.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:55.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5-clean 2026-02-20T21:12:55.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap5-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:12:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:12:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:12:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:12:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:12:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:12:55.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:12:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=9 2026-02-20T21:12:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:12:55.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:12:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 9869312 bytes 2026-02-20T21:12:56.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 49685504 bytes 2026-02-20T21:12:57.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 72166400 bytes 2026-02-20T21:12:57.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 164641792 bytes 2026-02-20T21:12:58.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 202659840 bytes 2026-02-20T21:12:59.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 262861824 bytes 2026-02-20T21:12:59.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 302202880 bytes 2026-02-20T21:13:00.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 321853440 bytes 2026-02-20T21:13:00.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 363954176 bytes 2026-02-20T21:13:01.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 415169536 bytes 2026-02-20T21:13:01.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 416304128 bytes 2026-02-20T21:13:01.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 426402816 bytes 2026-02-20T21:13:02.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 460643328 bytes 2026-02-20T21:13:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 517214208 bytes 2026-02-20T21:13:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 521428992 bytes 2026-02-20T21:13:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 562737152 bytes 2026-02-20T21:13:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 568148992 bytes 2026-02-20T21:13:04.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 586148864 bytes 2026-02-20T21:13:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:13:04.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:13:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:13:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:13:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:13:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:13:04.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:04.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:13:05.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:13:05.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:05.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:05.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:05.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:05.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:13:05.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:13:05.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:13:05.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:13:05.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:13:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:13:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:13:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-20T21:13:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":14592347.33,"entries_behind_primary":7363,"entries_per_second":922.17,"non_primary_position":{"entry_tid":54739,"object_number":215,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":217,"tag_tid":15},"seconds_until_synced":7} 2026-02-20T21:13:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.0 on trial163 2026-02-20T21:13:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:12:38 2026-02-20T21:13:05.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:13:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:13:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:13:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:13:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:12:38' 2026-02-20T21:13:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:13:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-20T21:13:05.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-20T21:13:05.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":14592347.33,"entries_behind_primary":7363,"entries_per_second":922.17,"non_primary_position":{"entry_tid":54739,"object_number":215,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":217,"tag_tid":15},"seconds_until_synced":7} 2026-02-20T21:13:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:13:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:13:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-20T21:13:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:13:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.0 on trial163 2026-02-20T21:13:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-20T21:13:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:12:38' 2026-02-20T21:13:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:12:38' 2026-02-20T21:13:05.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-20 21:12:38' 2026-02-20T21:13:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:13:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:13:05.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142044287mirror.0trial16320.2.0-677-gf7870454trueOK43304332mirror.2trial16320.2.0-677-gf7870454falseOK43484351mirror.3trial16320.2.0-677-gf7870454falseOK44334436mirror.1trial16320.2.0-677-gf7870454falseOKtestba923525-490d-4a76-8e5f-b742509a94daup+replayingreplaying, {"bytes_per_second":14592347.33,"entries_behind_primary":7363,"entries_per_second":922.17,"non_primary_position":{"entry_tid":54739,"object_number":215,"tag_tid":14},"primary_position":{"entry_tid":1,"object_number":217,"tag_tid":15},"seconds_until_synced":7}42044287mirror.0trial1632026-02-20 21:12:38cluster28b968eaf-6c24-469f-bea5-c8acb780219eup+stoppedlocal image is primary2026-02-20 21:12:38' 2026-02-20T21:13:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-20T21:13:05.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:12:38' 2026-02-20T21:13:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-20T21:13:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:13:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-20T21:13:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:13:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap6 2026-02-20T21:13:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap6 2026-02-20T21:13:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:13:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6 2026-02-20T21:13:05.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:13:27.430 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6972161255659224) > weight (0.25) 2026-02-20T21:13:27.430 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3198149150566406) > weight (0.25) 2026-02-20T21:13:27.430 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.4989761968203711) > weight (0.25) 2026-02-20T21:13:27.430 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.6002067510312434) > weight (0.25) 2026-02-20T21:13:27.430 INFO:tasks.rbd_mirror_thrash:waiting for 112.566013168129 secs before thrashing 2026-02-20T21:13:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:13:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:13:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:13:36.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:13:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:36.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:13:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:13:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:13:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:13:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:13:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:13:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:13:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:13:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:13:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:13:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:13:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:13:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:13:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:13:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:13:37.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:13:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:13:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:37.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:13:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:13:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:37.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:37.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:13:37.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:13:37.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:13:37.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:37.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:13:37.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:13:37.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:13:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:13:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:13:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:13:37.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:13:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:13:37.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:37.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:37.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:37.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:37.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:13:37.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:13:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:13:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:13:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:13:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:13:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:13:37.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:13:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:13:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:13:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:13:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:13:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:13:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:13:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:13:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:13:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:13:37.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:13:37.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:13:37.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:37.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:13:37.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:13:37.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:13:37.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:37.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:13:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:38.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:38.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:38.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:38.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-20T21:13:38.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:38.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:38.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:38.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=255, tag_tid=17, entry_tid=37331], [object_number=254, tag_tid=17, entry_tid=37330], [object_number=253, tag_tid=17, entry_tid=37329], [object_number=252, tag_tid=17, entry_tid=37328]]], state=connected] 2026-02-20T21:13:38.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:38.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:38.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:38.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:38.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-20T21:13:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:38.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:38.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:38.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=255, tag_tid=17, entry_tid=37331], [object_number=254, tag_tid=17, entry_tid=37330], [object_number=253, tag_tid=17, entry_tid=37329], [object_number=252, tag_tid=17, entry_tid=37328]]], state=connected] 2026-02-20T21:13:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=255, tag_tid=17, entry_tid=37331' 2026-02-20T21:13:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=255, tag_tid=17, entry_tid=37331' 2026-02-20T21:13:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=255, tag_tid=17, entry_tid=37331' '!=' '' 2026-02-20T21:13:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:13:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=255, tag_tid=17, entry_tid=37331' = '' 2026-02-20T21:13:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=255, tag_tid=17, entry_tid=37331' 2026-02-20T21:13:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-20T21:13:38.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=255, tag_tid=17, entry_tid=37331' 2026-02-20T21:13:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:38.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-20T21:13:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:38.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:13:38.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=255, tag_tid=17, entry_tid=37331' 2026-02-20T21:13:38.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:38.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:38.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37331 2026-02-20T21:13:38.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 37331 2026-02-20T21:13:38.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:13:38.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:38.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:13:38.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:38.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:38.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:38.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:38.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:38.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:38.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:38.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:38.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:38.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:38.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:38.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:38.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:38.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:38.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-20T21:13:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=254, tag_tid=17, entry_tid=37678], [object_number=253, tag_tid=17, entry_tid=37677], [object_number=252, tag_tid=17, entry_tid=37676], [object_number=255, tag_tid=17, entry_tid=37675]]], state=connected] 2026-02-20T21:13:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-20T21:13:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=254, tag_tid=17, entry_tid=37678], [object_number=253, tag_tid=17, entry_tid=37677], [object_number=252, tag_tid=17, entry_tid=37676], [object_number=255, tag_tid=17, entry_tid=37675]]], state=connected] 2026-02-20T21:13:38.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=254, tag_tid=17, entry_tid=37678' 2026-02-20T21:13:38.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=254, tag_tid=17, entry_tid=37678' 2026-02-20T21:13:38.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=254, tag_tid=17, entry_tid=37678' '!=' 'object_number=255, tag_tid=17, entry_tid=37331' 2026-02-20T21:13:38.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:13:38.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=254, tag_tid=17, entry_tid=37678' = 'object_number=255, tag_tid=17, entry_tid=37331' 2026-02-20T21:13:38.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=254, tag_tid=17, entry_tid=37678' 2026-02-20T21:13:38.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:38.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:38.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:38.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-20T21:13:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=254, tag_tid=17, entry_tid=37678' 2026-02-20T21:13:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-20T21:13:38.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:38.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:38.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:13:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=254, tag_tid=17, entry_tid=37678' 2026-02-20T21:13:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37678 2026-02-20T21:13:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 37678 2026-02-20T21:13:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:13:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:13:38.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:38.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:38.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:38.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:38.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:38.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:38.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:38.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-20T21:13:38.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:38.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:38.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:38.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=255, tag_tid=17, entry_tid=37987], [object_number=254, tag_tid=17, entry_tid=37986], [object_number=253, tag_tid=17, entry_tid=37985], [object_number=252, tag_tid=17, entry_tid=37984]]], state=connected] 2026-02-20T21:13:38.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:38.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:38.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:38.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.977 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.2760772227417341) > weight (0.25) 2026-02-20T21:13:38.978 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9693655621261191) > weight (0.25) 2026-02-20T21:13:38.978 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.739686124454111) > weight (0.25) 2026-02-20T21:13:38.978 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.6214776403451704) > weight (0.25) 2026-02-20T21:13:38.978 INFO:tasks.rbd_mirror_thrash:waiting for 109.5209821308837 secs before thrashing 2026-02-20T21:13:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-20T21:13:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=255, tag_tid=17, entry_tid=37987], [object_number=254, tag_tid=17, entry_tid=37986], [object_number=253, tag_tid=17, entry_tid=37985], [object_number=252, tag_tid=17, entry_tid=37984]]], state=connected] 2026-02-20T21:13:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:38.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=255, tag_tid=17, entry_tid=37987' 2026-02-20T21:13:38.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=255, tag_tid=17, entry_tid=37987' 2026-02-20T21:13:38.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=255, tag_tid=17, entry_tid=37987' '!=' 'object_number=254, tag_tid=17, entry_tid=37678' 2026-02-20T21:13:38.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:13:38.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=255, tag_tid=17, entry_tid=37987' = 'object_number=254, tag_tid=17, entry_tid=37678' 2026-02-20T21:13:38.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=255, tag_tid=17, entry_tid=37987' 2026-02-20T21:13:38.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:38.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-20T21:13:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=255, tag_tid=17, entry_tid=37987' 2026-02-20T21:13:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-20T21:13:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:38.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:13:38.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=255, tag_tid=17, entry_tid=37987' 2026-02-20T21:13:38.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37987 2026-02-20T21:13:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 37987 2026-02-20T21:13:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:13:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:13:39.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:39.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:39.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:39.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:39.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:39.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:39.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:39.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:39.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:39.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:39.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:39.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:39.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:39.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:39.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:39.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:39.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:39.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:39.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:39.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:39.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:39.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:39.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:39.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:39.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:39.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:39.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-20T21:13:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=253, tag_tid=17, entry_tid=38277], [object_number=252, tag_tid=17, entry_tid=38276], [object_number=255, tag_tid=17, entry_tid=38275], [object_number=254, tag_tid=17, entry_tid=38274]]], state=connected] 2026-02-20T21:13:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:39.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-20T21:13:39.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:39.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:39.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:39.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=253, tag_tid=17, entry_tid=38277], [object_number=252, tag_tid=17, entry_tid=38276], [object_number=255, tag_tid=17, entry_tid=38275], [object_number=254, tag_tid=17, entry_tid=38274]]], state=connected] 2026-02-20T21:13:39.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=253, tag_tid=17, entry_tid=38277' 2026-02-20T21:13:39.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=253, tag_tid=17, entry_tid=38277' 2026-02-20T21:13:39.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=253, tag_tid=17, entry_tid=38277' '!=' 'object_number=255, tag_tid=17, entry_tid=37987' 2026-02-20T21:13:39.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:13:39.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=253, tag_tid=17, entry_tid=38277' = 'object_number=255, tag_tid=17, entry_tid=37987' 2026-02-20T21:13:39.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=253, tag_tid=17, entry_tid=38277' 2026-02-20T21:13:39.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:39.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:39.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-20T21:13:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=253, tag_tid=17, entry_tid=38277' 2026-02-20T21:13:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:39.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:39.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-20T21:13:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:39.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:13:39.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=253, tag_tid=17, entry_tid=38277' 2026-02-20T21:13:39.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:39.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:39.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38277 2026-02-20T21:13:39.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 38277 2026-02-20T21:13:39.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:13:39.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:39.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:13:39.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:39.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:39.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:39.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:39.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:39.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:39.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:39.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:39.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:39.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:39.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:39.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:39.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:39.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:39.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:39.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:39.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:39.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:39.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:39.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:39.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:39.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:39.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:39.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:39.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:39.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:39.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:39.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:39.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:39.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:39.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:39.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:39.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:39.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:39.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-20T21:13:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:39.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=254, tag_tid=17, entry_tid=38582], [object_number=253, tag_tid=17, entry_tid=38581], [object_number=252, tag_tid=17, entry_tid=38580], [object_number=255, tag_tid=17, entry_tid=38579]]], state=connected] 2026-02-20T21:13:39.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:39.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:39.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:39.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:39.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:39.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 63 2026-02-20T21:13:39.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:39.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:39.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:39.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=254, tag_tid=17, entry_tid=38582], [object_number=253, tag_tid=17, entry_tid=38581], [object_number=252, tag_tid=17, entry_tid=38580], [object_number=255, tag_tid=17, entry_tid=38579]]], state=connected] 2026-02-20T21:13:39.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=254, tag_tid=17, entry_tid=38582' 2026-02-20T21:13:39.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=254, tag_tid=17, entry_tid=38582' 2026-02-20T21:13:39.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=254, tag_tid=17, entry_tid=38582' '!=' 'object_number=253, tag_tid=17, entry_tid=38277' 2026-02-20T21:13:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:13:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=254, tag_tid=17, entry_tid=38582' = 'object_number=253, tag_tid=17, entry_tid=38277' 2026-02-20T21:13:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=254, tag_tid=17, entry_tid=38582' 2026-02-20T21:13:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-20T21:13:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=254, tag_tid=17, entry_tid=38582' 2026-02-20T21:13:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-20T21:13:39.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:39.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:39.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:39.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:13:39.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=254, tag_tid=17, entry_tid=38582' 2026-02-20T21:13:39.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:39.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38582 2026-02-20T21:13:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 38582 2026-02-20T21:13:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:13:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:39.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:13:39.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:39.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:39.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:39.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:39.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:39.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:39.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:39.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:39.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:39.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:39.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:39.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:39.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:39.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:39.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:39.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:39.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:39.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:39.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:39.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:39.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:39.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=258, tag_tid=17, entry_tid=38874], [object_number=257, tag_tid=17, entry_tid=38873], [object_number=256, tag_tid=17, entry_tid=38872], [object_number=259, tag_tid=17, entry_tid=38871]]], state=connected] 2026-02-20T21:13:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:40.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=258, tag_tid=17, entry_tid=38874], [object_number=257, tag_tid=17, entry_tid=38873], [object_number=256, tag_tid=17, entry_tid=38872], [object_number=259, tag_tid=17, entry_tid=38871]]], state=connected] 2026-02-20T21:13:40.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=258, tag_tid=17, entry_tid=38874' 2026-02-20T21:13:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=258, tag_tid=17, entry_tid=38874' 2026-02-20T21:13:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=258, tag_tid=17, entry_tid=38874' '!=' 'object_number=254, tag_tid=17, entry_tid=38582' 2026-02-20T21:13:40.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:13:40.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=258, tag_tid=17, entry_tid=38874' = 'object_number=254, tag_tid=17, entry_tid=38582' 2026-02-20T21:13:40.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=258, tag_tid=17, entry_tid=38874' 2026-02-20T21:13:40.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:40.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:40.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:40.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-20T21:13:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=258, tag_tid=17, entry_tid=38874' 2026-02-20T21:13:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-20T21:13:40.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:40.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:40.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:40.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:13:40.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=258, tag_tid=17, entry_tid=38874' 2026-02-20T21:13:40.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:40.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38874 2026-02-20T21:13:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 38874 2026-02-20T21:13:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:13:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:40.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:13:40.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:40.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:40.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:40.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:40.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:40.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:40.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:40.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:40.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:40.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:40.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:40.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:40.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=258, tag_tid=17, entry_tid=39202], [object_number=257, tag_tid=17, entry_tid=39201], [object_number=256, tag_tid=17, entry_tid=39200], [object_number=259, tag_tid=17, entry_tid=39199]]], state=connected] 2026-02-20T21:13:40.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:40.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:40.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:40.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:40.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:40.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:40.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:40.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:40.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:40.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=258, tag_tid=17, entry_tid=39202], [object_number=257, tag_tid=17, entry_tid=39201], [object_number=256, tag_tid=17, entry_tid=39200], [object_number=259, tag_tid=17, entry_tid=39199]]], state=connected] 2026-02-20T21:13:40.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=258, tag_tid=17, entry_tid=39202' 2026-02-20T21:13:40.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=258, tag_tid=17, entry_tid=39202' 2026-02-20T21:13:40.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=258, tag_tid=17, entry_tid=39202' '!=' 'object_number=258, tag_tid=17, entry_tid=38874' 2026-02-20T21:13:40.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:13:40.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=258, tag_tid=17, entry_tid=39202' = 'object_number=258, tag_tid=17, entry_tid=38874' 2026-02-20T21:13:40.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=258, tag_tid=17, entry_tid=39202' 2026-02-20T21:13:40.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-20T21:13:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=258, tag_tid=17, entry_tid=39202' 2026-02-20T21:13:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-20T21:13:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:13:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=258, tag_tid=17, entry_tid=39202' 2026-02-20T21:13:40.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:40.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39202 2026-02-20T21:13:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 39202 2026-02-20T21:13:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:13:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:40.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:13:40.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:40.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:40.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:40.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:40.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:40.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:40.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:40.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:40.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:40.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:40.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:40.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:40.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:40.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:40.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:40.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:40.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:40.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:40.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:40.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:40.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:40.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:40.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:40.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:40.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:40.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:40.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=256, tag_tid=17, entry_tid=39524], [object_number=259, tag_tid=17, entry_tid=39523], [object_number=258, tag_tid=17, entry_tid=39522], [object_number=257, tag_tid=17, entry_tid=39521]]], state=connected] 2026-02-20T21:13:40.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:40.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:40.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:40.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:40.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:40.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:40.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:40.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=256, tag_tid=17, entry_tid=39524], [object_number=259, tag_tid=17, entry_tid=39523], [object_number=258, tag_tid=17, entry_tid=39522], [object_number=257, tag_tid=17, entry_tid=39521]]], state=connected] 2026-02-20T21:13:40.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=256, tag_tid=17, entry_tid=39524' 2026-02-20T21:13:40.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=256, tag_tid=17, entry_tid=39524' 2026-02-20T21:13:40.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=256, tag_tid=17, entry_tid=39524' '!=' 'object_number=258, tag_tid=17, entry_tid=39202' 2026-02-20T21:13:40.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:13:40.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=256, tag_tid=17, entry_tid=39524' = 'object_number=258, tag_tid=17, entry_tid=39202' 2026-02-20T21:13:40.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=256, tag_tid=17, entry_tid=39524' 2026-02-20T21:13:40.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:40.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:40.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:40.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-20T21:13:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=256, tag_tid=17, entry_tid=39524' 2026-02-20T21:13:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-20T21:13:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:40.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:40.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:13:40.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=256, tag_tid=17, entry_tid=39524' 2026-02-20T21:13:40.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:40.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:40.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39524 2026-02-20T21:13:40.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 39524 2026-02-20T21:13:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:13:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:13:40.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:40.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:40.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:40.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:40.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:40.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:40.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:40.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:40.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:40.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:40.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:40.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:40.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:41.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:41.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:41.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:41.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=259, tag_tid=17, entry_tid=39847], [object_number=258, tag_tid=17, entry_tid=39846], [object_number=257, tag_tid=17, entry_tid=39845], [object_number=256, tag_tid=17, entry_tid=39844]]], state=connected] 2026-02-20T21:13:41.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:41.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:41.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:41.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:41.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:41.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:41.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:41.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:41.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:41.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:41.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:41.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:41.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:41.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=259, tag_tid=17, entry_tid=39847], [object_number=258, tag_tid=17, entry_tid=39846], [object_number=257, tag_tid=17, entry_tid=39845], [object_number=256, tag_tid=17, entry_tid=39844]]], state=connected] 2026-02-20T21:13:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=259, tag_tid=17, entry_tid=39847' 2026-02-20T21:13:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=259, tag_tid=17, entry_tid=39847' 2026-02-20T21:13:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=259, tag_tid=17, entry_tid=39847' '!=' 'object_number=256, tag_tid=17, entry_tid=39524' 2026-02-20T21:13:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:13:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=259, tag_tid=17, entry_tid=39847' = 'object_number=256, tag_tid=17, entry_tid=39524' 2026-02-20T21:13:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=259, tag_tid=17, entry_tid=39847' 2026-02-20T21:13:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-20T21:13:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=259, tag_tid=17, entry_tid=39847' 2026-02-20T21:13:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-20T21:13:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:13:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=259, tag_tid=17, entry_tid=39847' 2026-02-20T21:13:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:41.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39847 2026-02-20T21:13:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 39847 2026-02-20T21:13:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:13:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:41.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:13:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:41.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:41.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:41.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:41.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:41.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:41.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:41.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:41.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:41.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:41.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:41.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:41.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:41.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=256, tag_tid=17, entry_tid=40208], [object_number=259, tag_tid=17, entry_tid=40207], [object_number=258, tag_tid=17, entry_tid=40206], [object_number=257, tag_tid=17, entry_tid=40205]]], state=connected] 2026-02-20T21:13:41.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:41.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:41.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:41.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:41.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:41.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:41.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=256, tag_tid=17, entry_tid=40208], [object_number=259, tag_tid=17, entry_tid=40207], [object_number=258, tag_tid=17, entry_tid=40206], [object_number=257, tag_tid=17, entry_tid=40205]]], state=connected] 2026-02-20T21:13:41.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=256, tag_tid=17, entry_tid=40208' 2026-02-20T21:13:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=256, tag_tid=17, entry_tid=40208' 2026-02-20T21:13:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=256, tag_tid=17, entry_tid=40208' '!=' 'object_number=259, tag_tid=17, entry_tid=39847' 2026-02-20T21:13:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:13:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=256, tag_tid=17, entry_tid=40208' = 'object_number=259, tag_tid=17, entry_tid=39847' 2026-02-20T21:13:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=256, tag_tid=17, entry_tid=40208' 2026-02-20T21:13:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-20T21:13:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=256, tag_tid=17, entry_tid=40208' 2026-02-20T21:13:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:41.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-20T21:13:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:13:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=256, tag_tid=17, entry_tid=40208' 2026-02-20T21:13:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40208 2026-02-20T21:13:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 40208 2026-02-20T21:13:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:13:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:13:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:41.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:41.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:41.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:41.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:41.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:41.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:41.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:41.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:41.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:41.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:41.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:41.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:41.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:41.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:41.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:41.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:41.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:41.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:41.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:41.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:41.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:41.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:41.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:41.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:41.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:41.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:41.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:41.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:41.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:41.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:41.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:41.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:41.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=256, tag_tid=17, entry_tid=40236], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234], [object_number=257, tag_tid=17, entry_tid=40233]]], state=connected] 2026-02-20T21:13:41.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:41.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:41.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:41.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:41.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:41.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:41.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=256, tag_tid=17, entry_tid=40236], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234], [object_number=257, tag_tid=17, entry_tid=40233]]], state=connected] 2026-02-20T21:13:41.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=256, tag_tid=17, entry_tid=40236' 2026-02-20T21:13:41.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=256, tag_tid=17, entry_tid=40236' 2026-02-20T21:13:41.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=256, tag_tid=17, entry_tid=40236' '!=' 'object_number=256, tag_tid=17, entry_tid=40208' 2026-02-20T21:13:41.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:13:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=256, tag_tid=17, entry_tid=40236' = 'object_number=256, tag_tid=17, entry_tid=40208' 2026-02-20T21:13:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=256, tag_tid=17, entry_tid=40236' 2026-02-20T21:13:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=18 2026-02-20T21:13:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=256, tag_tid=17, entry_tid=40236' 2026-02-20T21:13:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:13:41.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=17 2026-02-20T21:13:41.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:41.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:41.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:13:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=256, tag_tid=17, entry_tid=40236' 2026-02-20T21:13:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:13:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:13:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40236 2026-02-20T21:13:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 18 = 17 -a 1 -le 40236 2026-02-20T21:13:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:13:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:13:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:41.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:41.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:41.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:41.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:41.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:41.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:41.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:41.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:41.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:41.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:41.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:41.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:41.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=256, tag_tid=17, entry_tid=40236], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234], [object_number=257, tag_tid=17, entry_tid=40233]]], state=connected] 2026-02-20T21:13:42.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:42.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:42.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:42.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:42.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:42.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:42.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:42.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:42.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=256, tag_tid=17, entry_tid=40236], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234], [object_number=257, tag_tid=17, entry_tid=40233]]], state=connected] 2026-02-20T21:13:42.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:42.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=256, tag_tid=17, entry_tid=40236' 2026-02-20T21:13:42.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=256, tag_tid=17, entry_tid=40236' 2026-02-20T21:13:42.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=256, tag_tid=17, entry_tid=40236' '!=' 'object_number=256, tag_tid=17, entry_tid=40236' 2026-02-20T21:13:42.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:13:42.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:13:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:13:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:13:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:13:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:13:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:13:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:13:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:13:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:13:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:13:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:13:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:13:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:13:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:13:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:13:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:13:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:13:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:13:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:13:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:13:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:13:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:13:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:13:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:13:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:13:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:13:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:13:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:13:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:13:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:13:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:13:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:13:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:13:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:13:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:13:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:13:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:13:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:13:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:13:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:13:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:13:42.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:13:42.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:13:42.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:42.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:13:42.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:42.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:42.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:42.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:13:42.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:42.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:42.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:42.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:13:42.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:42.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:42.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:42.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:42.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:42.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:42.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:42.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:42.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:13:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:13:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:13:42.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:13:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:13:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:13:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:13:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:13:42.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:13:42.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:42.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:13:42.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:42.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 64 2026-02-20T21:13:42.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 64 2026-02-20T21:13:42.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:13:42.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:42.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=257, tag_tid=18, entry_tid=1], [object_number=256, tag_tid=18, entry_tid=0], [object_number=259, tag_tid=17, entry_tid=40235], [object_number=258, tag_tid=17, entry_tid=40234]]], state=connected] 2026-02-20T21:13:42.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:13:42.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:42.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=257, tag_tid=18, entry_tid=1' -a 'object_number=257, tag_tid=18, entry_tid=1' = 'object_number=257, tag_tid=18, entry_tid=1' 2026-02-20T21:13:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap6 2026-02-20T21:13:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6 2026-02-20T21:13:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:13:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:13:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:13:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap6 2026-02-20T21:13:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-20T21:13:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:13:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:13:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 11 2026-02-20T21:13:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ba958b175d 2026-02-20T21:13:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ba958b175d 2026-02-20T21:13:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:13:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:13:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:13:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:13:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:06:40 2026 2026-02-20T21:13:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:12:53 2026 2026-02-20T21:13:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:13:30 2026 2026-02-20T21:13:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:13:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ba958b175d 2026-02-20T21:13:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:13:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:13:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:13:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:13:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap5-clean ']' 2026-02-20T21:13:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap5-clean 2026-02-20T21:13:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5-clean 2026-02-20T21:13:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:13:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:13:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:13:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:13:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:13:44.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:13:44.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:13:44.050 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.047+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:13:44.053 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.050+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:44.053 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.050+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:44.053 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.050+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:44.053 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.050+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:44.053 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.050+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:44.164 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.161+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:13:44.238 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.235+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:13:44.240 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.236+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:13:44.240 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.237+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:13:44.240 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.237+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:13:44.242 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.239+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:44.242 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.239+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:44.317 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.314+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:13:44.317 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.314+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:13:44.318 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:44.315+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:13:44.390 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:13:44.387+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:13:44.390 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:13:44.387+0000 7f412dcce640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:13:44.461 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:13:44.458+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:13:44.461 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:13:44.458+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:13:44.461 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:13:44.458+0000 7f412ecd0640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:13:44.462 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:13:44.459+0000 7f412fcd2640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:13:44.462 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:13:44.459+0000 7f41304d3640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:13:44.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:13:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:13:44.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:13:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap6 2026-02-20T21:13:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:44.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6 2026-02-20T21:13:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:13:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:13:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:13:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:13:44.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:13:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:13:45.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:13:45.123 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.119+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:45.123 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.119+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:45.123 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.119+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:45.123 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.119+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:45.123 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.119+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:45.123 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.119+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:45.255 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.252+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:13:45.257 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.254+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:13:45.258 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.255+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:45.258 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.255+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:45.258 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.255+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:45.258 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.255+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:13:45.289 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.286+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:13:45.346 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.343+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:13:45.346 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.343+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:13:45.346 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.343+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:13:45.346 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:45.343+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:13:45.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:13:45.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:13:45.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:13:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap6-clean 2026-02-20T21:13:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap6-clean 2026-02-20T21:13:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:13:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6-clean 2026-02-20T21:13:45.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap6-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:13:45.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:13:45.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:13:45.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:13:45.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:45.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:45.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:13:45.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=5 2026-02-20T21:13:45.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:13:45.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:13:46.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 22709248 bytes 2026-02-20T21:13:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 46922752 bytes 2026-02-20T21:13:47.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 84014080 bytes 2026-02-20T21:13:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 112116736 bytes 2026-02-20T21:13:48.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 137798656 bytes 2026-02-20T21:13:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 152167424 bytes 2026-02-20T21:13:49.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 178598912 bytes 2026-02-20T21:13:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 246704128 bytes 2026-02-20T21:13:50.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 260898816 bytes 2026-02-20T21:13:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 298224640 bytes 2026-02-20T21:13:50.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:13:50.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:13:50.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:13:50.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:50.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:13:50.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:50.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:50.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:50.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:50.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:13:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:13:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:13:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:13:51.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:13:51.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:51.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:51.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:51.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:51.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:13:51.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:13:51.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:13:51.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:13:51.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:13:51.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:13:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:13:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-20T21:13:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":14865278.4,"entries_behind_primary":2559,"entries_per_second":936.83,"non_primary_position":{"entry_tid":37678,"object_number":254,"tag_tid":17},"primary_position":{"entry_tid":1,"object_number":257,"tag_tid":18},"seconds_until_synced":2} 2026-02-20T21:13:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.0 on trial163 2026-02-20T21:13:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:13:38 2026-02-20T21:13:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:13:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:13:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:13:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:13:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:13:38' 2026-02-20T21:13:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:13:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-20T21:13:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-20T21:13:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":14865278.4,"entries_behind_primary":2559,"entries_per_second":936.83,"non_primary_position":{"entry_tid":37678,"object_number":254,"tag_tid":17},"primary_position":{"entry_tid":1,"object_number":257,"tag_tid":18},"seconds_until_synced":2} 2026-02-20T21:13:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:13:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:13:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-20T21:13:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:13:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:13:51.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.0 on trial163 2026-02-20T21:13:51.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-20T21:13:51.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:13:51.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:13:38' 2026-02-20T21:13:51.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:13:38' 2026-02-20T21:13:51.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:13:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:13:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-20 21:13:38' 2026-02-20T21:13:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:13:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:13:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:13:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142044287mirror.0trial16320.2.0-677-gf7870454trueOK43304332mirror.2trial16320.2.0-677-gf7870454falseOK43484351mirror.3trial16320.2.0-677-gf7870454falseOK44334436mirror.1trial16320.2.0-677-gf7870454falseOKtestba923525-490d-4a76-8e5f-b742509a94daup+replayingreplaying, {"bytes_per_second":14865278.4,"entries_behind_primary":2559,"entries_per_second":936.83,"non_primary_position":{"entry_tid":37678,"object_number":254,"tag_tid":17},"primary_position":{"entry_tid":1,"object_number":257,"tag_tid":18},"seconds_until_synced":2}42044287mirror.0trial1632026-02-20 21:13:38cluster28b968eaf-6c24-469f-bea5-c8acb780219eup+stoppedlocal image is primary2026-02-20 21:13:38' 2026-02-20T21:13:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-20T21:13:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:13:52.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:13:38' 2026-02-20T21:13:52.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:13:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-20T21:13:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:13:52.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-20T21:13:52.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:13:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:13:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap7 2026-02-20T21:13:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap7 2026-02-20T21:13:52.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:13:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:13:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:13:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7 2026-02-20T21:13:52.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:13:57.806 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:57.803+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe65b0 2026-02-20T21:13:57.806 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:13:57.803+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:14:22.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:14:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:14:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:14:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:14:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:22.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:14:23.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:14:23.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:23.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:23.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:23.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:14:23.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:14:23.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:14:23.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:14:23.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:14:23.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:14:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:14:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:14:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:14:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:14:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:14:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:14:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:14:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:14:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:14:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:14:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:14:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:14:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:14:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:14:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:14:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:14:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:14:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:14:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:14:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:14:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:14:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:14:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:14:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:14:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:14:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:14:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:14:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:14:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:14:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:14:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:14:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:14:23.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:14:23.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:14:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:23.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:14:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:14:24.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:14:24.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:14:24.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:14:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:14:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:14:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:14:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:14:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:14:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:14:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:14:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:14:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:14:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:14:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:14:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:14:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:14:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:14:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:14:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:14:24.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:14:24.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:14:24.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:14:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:14:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:14:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:14:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:14:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:14:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:14:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:14:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:14:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:14:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:14:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:14:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:14:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:14:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:14:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:14:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:14:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:14:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:14:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:14:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:14:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:14:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:14:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:14:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:14:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:14:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:14:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:14:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:14:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:14:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:14:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:14:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:14:24.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:14:24.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:24.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:24.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:24.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:14:24.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:14:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:14:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:24.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:14:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:14:24.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:24.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:24.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:14:24.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:24.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 68 2026-02-20T21:14:24.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 68 2026-02-20T21:14:24.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:14:24.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:24.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=274, tag_tid=20, entry_tid=18502], [object_number=273, tag_tid=20, entry_tid=18501], [object_number=272, tag_tid=20, entry_tid=18500], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:24.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=273, tag_tid=21, entry_tid=1' 2026-02-20T21:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:14:24.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:14:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:14:24.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:14:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:24.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 68 2026-02-20T21:14:24.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 68 2026-02-20T21:14:24.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:14:24.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:24.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=274, tag_tid=20, entry_tid=18502], [object_number=273, tag_tid=20, entry_tid=18501], [object_number=272, tag_tid=20, entry_tid=18500], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:24.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:24.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:24.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=273, tag_tid=21, entry_tid=1' -a 'object_number=273, tag_tid=21, entry_tid=1' = 'object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:24.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=274, tag_tid=20, entry_tid=18502' '!=' '' 2026-02-20T21:14:24.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:14:24.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=274, tag_tid=20, entry_tid=18502' = '' 2026-02-20T21:14:24.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:24.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=273, tag_tid=21, entry_tid=1' 2026-02-20T21:14:24.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:14:24.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:14:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=21 2026-02-20T21:14:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:14:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:14:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=20 2026-02-20T21:14:24.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=273, tag_tid=21, entry_tid=1' 2026-02-20T21:14:24.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:14:24.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:14:24.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:14:24.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:24.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:14:24.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:14:24.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=18502 2026-02-20T21:14:24.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 21 = 20 -a 1 -le 18502 2026-02-20T21:14:24.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:14:24.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:14:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:14:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:14:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:24.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:14:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:14:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:14:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:14:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:14:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:14:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:14:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:14:24.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:24.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:14:24.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:14:24.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:14:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:24.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:14:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:14:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:14:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:14:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:14:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:14:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:14:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:14:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:14:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:14:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:14:24.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:14:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:14:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:14:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:14:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:14:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:14:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:14:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:14:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:14:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:24.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:24.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:24.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:14:24.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:14:24.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:14:24.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:14:24.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:14:24.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:24.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:24.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:24.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:14:24.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:24.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:14:24.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:24.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:24.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:24.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:14:24.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:24.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:24.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:24.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:14:24.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:24.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:14:24.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:14:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 68 2026-02-20T21:14:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 68 2026-02-20T21:14:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:14:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=274, tag_tid=20, entry_tid=18502], [object_number=273, tag_tid=20, entry_tid=18501], [object_number=272, tag_tid=20, entry_tid=18500], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=273, tag_tid=21, entry_tid=1' 2026-02-20T21:14:24.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:14:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:14:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:14:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:14:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:14:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 68 2026-02-20T21:14:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 68 2026-02-20T21:14:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:14:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=274, tag_tid=20, entry_tid=18502], [object_number=273, tag_tid=20, entry_tid=18501], [object_number=272, tag_tid=20, entry_tid=18500], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:24.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=273, tag_tid=21, entry_tid=1' -a 'object_number=273, tag_tid=21, entry_tid=1' = 'object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=274, tag_tid=20, entry_tid=18502' '!=' 'object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:14:24.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:14:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:14:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:14:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:14:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:14:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:14:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:14:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:14:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:14:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:14:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:14:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:14:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:14:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:14:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:14:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:14:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:14:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:14:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:14:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:14:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:14:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:14:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:14:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:14:25.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:14:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:14:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:14:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:14:25.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:14:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:25.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:14:25.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:14:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:14:25.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:25.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:25.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:25.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:14:25.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:25.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:14:25.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:14:25.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 68 2026-02-20T21:14:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 68 2026-02-20T21:14:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:14:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=274, tag_tid=20, entry_tid=18502], [object_number=273, tag_tid=20, entry_tid=18501], [object_number=272, tag_tid=20, entry_tid=18500], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:25.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=273, tag_tid=21, entry_tid=1' 2026-02-20T21:14:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:14:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:14:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:14:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:14:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:14:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 68 2026-02-20T21:14:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 68 2026-02-20T21:14:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:14:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=274, tag_tid=20, entry_tid=18502], [object_number=273, tag_tid=20, entry_tid=18501], [object_number=272, tag_tid=20, entry_tid=18500], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=273, tag_tid=21, entry_tid=1' -a 'object_number=273, tag_tid=21, entry_tid=1' = 'object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=274, tag_tid=20, entry_tid=18502' '!=' 'object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:14:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:14:26.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:14:26.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:26.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:26.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:26.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:14:26.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:14:26.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:14:26.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:14:26.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:14:26.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:14:26.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:26.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:14:26.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:14:26.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:26.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:14:26.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:14:26.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:14:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:14:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:14:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:14:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:14:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:14:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:14:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:14:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:14:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:14:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:14:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:14:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:14:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:14:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:14:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:14:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:14:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:14:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:14:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:14:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:14:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:14:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:14:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:14:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:14:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:14:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:14:26.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:14:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:14:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:14:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:14:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:14:26.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 68 2026-02-20T21:14:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 68 2026-02-20T21:14:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:14:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:26.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=274, tag_tid=20, entry_tid=18502], [object_number=273, tag_tid=20, entry_tid=18501], [object_number=272, tag_tid=20, entry_tid=18500], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:26.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:26.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=273, tag_tid=21, entry_tid=1' 2026-02-20T21:14:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:14:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:14:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:14:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:26.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:14:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:14:26.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 68 2026-02-20T21:14:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 68 2026-02-20T21:14:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:14:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:26.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=274, tag_tid=20, entry_tid=18502], [object_number=273, tag_tid=20, entry_tid=18501], [object_number=272, tag_tid=20, entry_tid=18500], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:26.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=273, tag_tid=21, entry_tid=1' -a 'object_number=273, tag_tid=21, entry_tid=1' = 'object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=274, tag_tid=20, entry_tid=18502' '!=' 'object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:14:26.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:14:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:14:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:27.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:14:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:14:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:14:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:14:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:14:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:14:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:14:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:14:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:14:27.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:14:27.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:14:27.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:27.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:14:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:14:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:14:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:14:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:14:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:14:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:14:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:14:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:14:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:14:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:27.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:14:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:14:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:14:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:14:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:14:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:14:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:14:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:14:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:14:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:14:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:14:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 68 2026-02-20T21:14:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 68 2026-02-20T21:14:28.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:14:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:28.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=274, tag_tid=20, entry_tid=18502], [object_number=273, tag_tid=20, entry_tid=18501], [object_number=272, tag_tid=20, entry_tid=18500], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:28.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:28.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=273, tag_tid=21, entry_tid=1' 2026-02-20T21:14:28.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:14:28.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:14:28.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:28.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:28.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:14:28.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:28.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 68 2026-02-20T21:14:28.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 68 2026-02-20T21:14:28.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:14:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:28.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=274, tag_tid=20, entry_tid=18502], [object_number=273, tag_tid=20, entry_tid=18501], [object_number=272, tag_tid=20, entry_tid=18500], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:28.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:28.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:28.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=273, tag_tid=21, entry_tid=1' -a 'object_number=273, tag_tid=21, entry_tid=1' = 'object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=274, tag_tid=20, entry_tid=18502' '!=' 'object_number=274, tag_tid=20, entry_tid=18502' 2026-02-20T21:14:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:14:28.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:14:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:14:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:14:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:14:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:14:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:14:30.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:14:30.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:14:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:14:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:14:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:30.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:14:30.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:30.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:30.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:14:30.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 68 2026-02-20T21:14:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 68 2026-02-20T21:14:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:14:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:30.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:30.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:30.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=273, tag_tid=21, entry_tid=1' 2026-02-20T21:14:30.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:14:30.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:30.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:30.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:30.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:14:30.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:14:30.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:14:30.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:14:30.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:14:30.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:14:30.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:14:30.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:14:30.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:30.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:14:30.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:30.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 68 2026-02-20T21:14:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 68 2026-02-20T21:14:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:14:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:30.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=273, tag_tid=21, entry_tid=1], [object_number=272, tag_tid=21, entry_tid=0], [object_number=274, tag_tid=20, entry_tid=18502], [object_number=275, tag_tid=20, entry_tid=18499]]], state=connected] 2026-02-20T21:14:30.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:14:30.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=273, tag_tid=21, entry_tid=1' 2026-02-20T21:14:30.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=273, tag_tid=21, entry_tid=1' -a 'object_number=273, tag_tid=21, entry_tid=1' = 'object_number=273, tag_tid=21, entry_tid=1' 2026-02-20T21:14:30.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap7 2026-02-20T21:14:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7 2026-02-20T21:14:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:14:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:14:30.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:14:31.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap7 2026-02-20T21:14:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-20T21:14:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:14:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:14:31.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 13 2026-02-20T21:14:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ba958b175d 2026-02-20T21:14:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ba958b175d 2026-02-20T21:14:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:14:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:14:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:14:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:14:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:06:40 2026 2026-02-20T21:14:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:12:53 2026 2026-02-20T21:14:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:13:30 2026 2026-02-20T21:14:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:14:31.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ba958b175d 2026-02-20T21:14:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:14:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:14:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:14:31.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:14:31.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:31.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap6-clean ']' 2026-02-20T21:14:31.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap6-clean 2026-02-20T21:14:31.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:31.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:31.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6-clean 2026-02-20T21:14:31.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:14:31.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:14:31.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:14:31.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:14:31.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:14:31.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:14:31.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:14:31.568 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.564+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:31.568 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.564+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:31.568 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.564+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:31.568 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.564+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:31.568 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.564+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:31.569 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.565+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:31.702 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.699+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:31.704 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.701+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:31.705 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.702+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:31.706 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.703+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:14:31.707 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.704+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:14:31.707 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.704+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:14:31.737 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.734+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:14:31.738 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.734+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:14:31.738 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.735+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:31.739 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.735+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:31.739 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.736+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:31.771 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.768+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:31.772 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.768+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:31.772 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.768+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:31.794 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.791+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:14:31.795 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.791+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:31.795 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.791+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:31.795 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.791+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:31.795 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.792+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:31.822 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.819+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:31.823 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.819+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:14:31.824 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.820+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:31.824 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.820+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:31.824 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.820+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:31.824 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:31.821+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:14:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:14:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:14:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap7 2026-02-20T21:14:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7 2026-02-20T21:14:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:14:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:14:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:14:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:14:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:14:32.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:14:32.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:14:32.689 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.686+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:32.690 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.686+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:32.690 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.686+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:32.692 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.688+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:14:32.692 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.688+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:14:32.692 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.688+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:14:32.789 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.785+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:32.789 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.786+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:32.859 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.856+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:32.860 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.857+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:32.863 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.860+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:14:32.864 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.860+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:14:32.864 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.860+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:14:32.864 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.861+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:14:32.897 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.894+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:32.897 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.894+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:32.897 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.894+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:32.898 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.895+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:32.899 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.895+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:32.966 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.963+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:32.966 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.963+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:32.966 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.963+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:32.966 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.964+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:14:32.968 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:32.965+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:33.004 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:33.001+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:14:33.005 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:33.003+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:33.005 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:33.003+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:33.006 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:33.003+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:33.006 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:14:33.003+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:14:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:14:33.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:14:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:14:33.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap7-clean 2026-02-20T21:14:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap7-clean 2026-02-20T21:14:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:14:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7-clean 2026-02-20T21:14:33.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap7-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:14:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:14:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:14:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:14:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:33.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:14:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-20T21:14:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:14:33.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:14:34.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 11556864 bytes 2026-02-20T21:14:35.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 58607616 bytes 2026-02-20T21:14:35.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 96163840 bytes 2026-02-20T21:14:35.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 111270912 bytes 2026-02-20T21:14:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 184057856 bytes 2026-02-20T21:14:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 238587904 bytes 2026-02-20T21:14:39.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 324189184 bytes 2026-02-20T21:14:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 334208000 bytes 2026-02-20T21:14:39.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 388967424 bytes 2026-02-20T21:14:40.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 424872960 bytes 2026-02-20T21:14:40.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 457973760 bytes 2026-02-20T21:14:41.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 521516032 bytes 2026-02-20T21:14:41.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 573314048 bytes 2026-02-20T21:14:41.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:14:41.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:14:41.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:14:41.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:41.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:14:41.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:41.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:41.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:41.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:41.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:14:41.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:14:41.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:14:41.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:14:42.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:14:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:14:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:14:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:14:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:14:42.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:14:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:14:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:14:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-20T21:14:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":1424094.9,"entries_behind_primary":1,"entries_per_second":91.2,"non_primary_position":{"entry_tid":1,"object_number":273,"tag_tid":21},"primary_position":{"entry_tid":1,"object_number":273,"tag_tid":22},"seconds_until_synced":0} 2026-02-20T21:14:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.0 on trial163 2026-02-20T21:14:42.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:14:38 2026-02-20T21:14:42.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:14:42.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:14:42.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:14:42.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:14:42.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:14:38' 2026-02-20T21:14:42.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:14:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-20T21:14:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-20T21:14:42.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":1424094.9,"entries_behind_primary":1,"entries_per_second":91.2,"non_primary_position":{"entry_tid":1,"object_number":273,"tag_tid":21},"primary_position":{"entry_tid":1,"object_number":273,"tag_tid":22},"seconds_until_synced":0} 2026-02-20T21:14:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:14:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:14:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-20T21:14:42.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:14:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:14:42.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.0 on trial163 2026-02-20T21:14:42.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-20T21:14:42.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:14:42.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:14:38' 2026-02-20T21:14:42.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:14:38' 2026-02-20T21:14:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:14:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:14:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-20 21:14:38' 2026-02-20T21:14:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:14:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:14:42.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:14:43.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142044287mirror.0trial16320.2.0-677-gf7870454trueOK43304332mirror.2trial16320.2.0-677-gf7870454falseOK43484351mirror.3trial16320.2.0-677-gf7870454falseOK44334436mirror.1trial16320.2.0-677-gf7870454falseOKtestba923525-490d-4a76-8e5f-b742509a94daup+replayingreplaying, {"bytes_per_second":1424094.9,"entries_behind_primary":1,"entries_per_second":91.2,"non_primary_position":{"entry_tid":1,"object_number":273,"tag_tid":21},"primary_position":{"entry_tid":1,"object_number":273,"tag_tid":22},"seconds_until_synced":0}42044287mirror.0trial1632026-02-20 21:14:38cluster28b968eaf-6c24-469f-bea5-c8acb780219eup+stoppedlocal image is primary2026-02-20 21:14:38' 2026-02-20T21:14:43.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-20T21:14:43.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:14:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:14:38' 2026-02-20T21:14:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:14:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-20T21:14:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:14:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-20T21:14:43.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:14:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:14:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap8 2026-02-20T21:14:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap8 2026-02-20T21:14:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:14:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:14:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:14:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8 2026-02-20T21:14:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:14:54.409 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:14:54.406+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e188f110 2026-02-20T21:14:54.411 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:14:54.408+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb5f0 2026-02-20T21:14:54.411 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:14:54.408+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc8a90 2026-02-20T21:15:13.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:15:13.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:13.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:13.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:13.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:15:13.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:15:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:15:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:13.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:15:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:15:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:15:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:15:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:15:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:15:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:15:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:15:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:15:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:15:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:15:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:15:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:15:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:15:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:15:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:15:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:15:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:14.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:15:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:15:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:15:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:15:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:15:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:15:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:15:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:15:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:15:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:15:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:15:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:15:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:15:14.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:14.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:15:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:15:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:15:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:15:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:15:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:15:15.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:15:15.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:15:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:15:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:15:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:15:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:15:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:15:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:15:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:15:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:15:15.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:15:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:15:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:15:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:15:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:15:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:15.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:15.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:15.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:15.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:15.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:15.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:15.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:15.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:15.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-20T21:15:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=305, tag_tid=23, entry_tid=32949], [object_number=304, tag_tid=23, entry_tid=32948], [object_number=307, tag_tid=23, entry_tid=32947], [object_number=306, tag_tid=23, entry_tid=32946]]], state=connected] 2026-02-20T21:15:15.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-20T21:15:15.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:15.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:15.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:15.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=305, tag_tid=23, entry_tid=32949], [object_number=304, tag_tid=23, entry_tid=32948], [object_number=307, tag_tid=23, entry_tid=32947], [object_number=306, tag_tid=23, entry_tid=32946]]], state=connected] 2026-02-20T21:15:15.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=305, tag_tid=23, entry_tid=32949' 2026-02-20T21:15:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=305, tag_tid=23, entry_tid=32949' 2026-02-20T21:15:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=305, tag_tid=23, entry_tid=32949' '!=' '' 2026-02-20T21:15:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=305, tag_tid=23, entry_tid=32949' = '' 2026-02-20T21:15:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=305, tag_tid=23, entry_tid=32949' 2026-02-20T21:15:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:15.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=305, tag_tid=23, entry_tid=32949' 2026-02-20T21:15:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:15.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:15.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:15.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=305, tag_tid=23, entry_tid=32949' 2026-02-20T21:15:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32949 2026-02-20T21:15:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 32949 2026-02-20T21:15:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:15.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:15.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:15.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:15.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:15.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:15.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:15.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:15.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:15.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:15.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:15.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:15.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:15.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-20T21:15:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:15.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=307, tag_tid=23, entry_tid=33219], [object_number=306, tag_tid=23, entry_tid=33218], [object_number=305, tag_tid=23, entry_tid=33217], [object_number=304, tag_tid=23, entry_tid=33216]]], state=connected] 2026-02-20T21:15:15.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:15.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:15.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:15.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:15.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:15.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:15.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:15.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:15.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:15.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:15.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:15.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:15.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:15.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:15.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-20T21:15:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:15.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=307, tag_tid=23, entry_tid=33219], [object_number=306, tag_tid=23, entry_tid=33218], [object_number=305, tag_tid=23, entry_tid=33217], [object_number=304, tag_tid=23, entry_tid=33216]]], state=connected] 2026-02-20T21:15:15.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=307, tag_tid=23, entry_tid=33219' 2026-02-20T21:15:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=307, tag_tid=23, entry_tid=33219' 2026-02-20T21:15:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=307, tag_tid=23, entry_tid=33219' '!=' 'object_number=305, tag_tid=23, entry_tid=32949' 2026-02-20T21:15:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=307, tag_tid=23, entry_tid=33219' = 'object_number=305, tag_tid=23, entry_tid=32949' 2026-02-20T21:15:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=307, tag_tid=23, entry_tid=33219' 2026-02-20T21:15:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:15.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:15.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:15.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:15.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=307, tag_tid=23, entry_tid=33219' 2026-02-20T21:15:15.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:15.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:15.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:15.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:15.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:15.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=307, tag_tid=23, entry_tid=33219' 2026-02-20T21:15:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33219 2026-02-20T21:15:15.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 33219 2026-02-20T21:15:15.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:15.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:15.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:15.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:15.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:15.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:15.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:15.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:15.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:15.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:15.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:15.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-20T21:15:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=304, tag_tid=23, entry_tid=33524], [object_number=307, tag_tid=23, entry_tid=33523], [object_number=306, tag_tid=23, entry_tid=33522], [object_number=305, tag_tid=23, entry_tid=33521]]], state=connected] 2026-02-20T21:15:16.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-20T21:15:16.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:16.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:16.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:16.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=304, tag_tid=23, entry_tid=33524], [object_number=307, tag_tid=23, entry_tid=33523], [object_number=306, tag_tid=23, entry_tid=33522], [object_number=305, tag_tid=23, entry_tid=33521]]], state=connected] 2026-02-20T21:15:16.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=304, tag_tid=23, entry_tid=33524' 2026-02-20T21:15:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=304, tag_tid=23, entry_tid=33524' 2026-02-20T21:15:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=304, tag_tid=23, entry_tid=33524' '!=' 'object_number=307, tag_tid=23, entry_tid=33219' 2026-02-20T21:15:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=304, tag_tid=23, entry_tid=33524' = 'object_number=307, tag_tid=23, entry_tid=33219' 2026-02-20T21:15:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=304, tag_tid=23, entry_tid=33524' 2026-02-20T21:15:16.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:16.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:16.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:16.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=304, tag_tid=23, entry_tid=33524' 2026-02-20T21:15:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:16.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:16.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:16.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=304, tag_tid=23, entry_tid=33524' 2026-02-20T21:15:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:16.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33524 2026-02-20T21:15:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 33524 2026-02-20T21:15:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:16.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:16.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:16.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:16.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:16.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:16.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:16.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:16.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:16.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:16.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-20T21:15:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:16.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:16.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=307, tag_tid=23, entry_tid=33807], [object_number=306, tag_tid=23, entry_tid=33806], [object_number=305, tag_tid=23, entry_tid=33805], [object_number=304, tag_tid=23, entry_tid=33804]]], state=connected] 2026-02-20T21:15:16.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:16.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:16.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:16.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 76 2026-02-20T21:15:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:16.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=307, tag_tid=23, entry_tid=33807], [object_number=306, tag_tid=23, entry_tid=33806], [object_number=305, tag_tid=23, entry_tid=33805], [object_number=304, tag_tid=23, entry_tid=33804]]], state=connected] 2026-02-20T21:15:16.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=307, tag_tid=23, entry_tid=33807' 2026-02-20T21:15:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=307, tag_tid=23, entry_tid=33807' 2026-02-20T21:15:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=307, tag_tid=23, entry_tid=33807' '!=' 'object_number=304, tag_tid=23, entry_tid=33524' 2026-02-20T21:15:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=307, tag_tid=23, entry_tid=33807' = 'object_number=304, tag_tid=23, entry_tid=33524' 2026-02-20T21:15:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=307, tag_tid=23, entry_tid=33807' 2026-02-20T21:15:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:16.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:16.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=307, tag_tid=23, entry_tid=33807' 2026-02-20T21:15:16.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:16.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:16.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:16.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:16.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=307, tag_tid=23, entry_tid=33807' 2026-02-20T21:15:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33807 2026-02-20T21:15:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 33807 2026-02-20T21:15:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:16.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:16.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:16.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:16.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:16.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:16.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:16.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:16.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:16.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:16.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:16.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:16.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:16.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:16.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:16.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:16.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:16.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:16.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:16.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:16.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:16.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:16.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:16.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=308, tag_tid=23, entry_tid=34044], [object_number=311, tag_tid=23, entry_tid=34043], [object_number=310, tag_tid=23, entry_tid=34042], [object_number=309, tag_tid=23, entry_tid=34041]]], state=connected] 2026-02-20T21:15:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:16.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:16.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:16.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:16.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:16.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:16.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:16.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:16.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:16.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=308, tag_tid=23, entry_tid=34044], [object_number=311, tag_tid=23, entry_tid=34043], [object_number=310, tag_tid=23, entry_tid=34042], [object_number=309, tag_tid=23, entry_tid=34041]]], state=connected] 2026-02-20T21:15:16.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=308, tag_tid=23, entry_tid=34044' 2026-02-20T21:15:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=308, tag_tid=23, entry_tid=34044' 2026-02-20T21:15:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=308, tag_tid=23, entry_tid=34044' '!=' 'object_number=307, tag_tid=23, entry_tid=33807' 2026-02-20T21:15:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=308, tag_tid=23, entry_tid=34044' = 'object_number=307, tag_tid=23, entry_tid=33807' 2026-02-20T21:15:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=308, tag_tid=23, entry_tid=34044' 2026-02-20T21:15:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=308, tag_tid=23, entry_tid=34044' 2026-02-20T21:15:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:16.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=308, tag_tid=23, entry_tid=34044' 2026-02-20T21:15:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34044 2026-02-20T21:15:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 34044 2026-02-20T21:15:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:16.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:16.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:16.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:16.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:16.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:16.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:16.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:16.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:16.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:16.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:16.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:16.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:16.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:16.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:16.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:16.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:16.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:16.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:17.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=308, tag_tid=23, entry_tid=34344], [object_number=311, tag_tid=23, entry_tid=34343], [object_number=310, tag_tid=23, entry_tid=34342], [object_number=309, tag_tid=23, entry_tid=34341]]], state=connected] 2026-02-20T21:15:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:17.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:17.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:17.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:17.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:17.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:17.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:17.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:17.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:17.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:17.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:17.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=308, tag_tid=23, entry_tid=34344], [object_number=311, tag_tid=23, entry_tid=34343], [object_number=310, tag_tid=23, entry_tid=34342], [object_number=309, tag_tid=23, entry_tid=34341]]], state=connected] 2026-02-20T21:15:17.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=308, tag_tid=23, entry_tid=34344' 2026-02-20T21:15:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=308, tag_tid=23, entry_tid=34344' 2026-02-20T21:15:17.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=308, tag_tid=23, entry_tid=34344' '!=' 'object_number=308, tag_tid=23, entry_tid=34044' 2026-02-20T21:15:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=308, tag_tid=23, entry_tid=34344' = 'object_number=308, tag_tid=23, entry_tid=34044' 2026-02-20T21:15:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=308, tag_tid=23, entry_tid=34344' 2026-02-20T21:15:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:17.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:17.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=308, tag_tid=23, entry_tid=34344' 2026-02-20T21:15:17.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:17.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:17.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:17.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:17.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:17.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:17.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=308, tag_tid=23, entry_tid=34344' 2026-02-20T21:15:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:17.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34344 2026-02-20T21:15:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 34344 2026-02-20T21:15:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:17.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:17.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:17.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:17.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:17.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:17.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:17.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:17.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:17.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:17.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:17.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:17.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:17.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:17.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:17.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:17.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:17.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=309, tag_tid=23, entry_tid=34629], [object_number=308, tag_tid=23, entry_tid=34628], [object_number=311, tag_tid=23, entry_tid=34627], [object_number=310, tag_tid=23, entry_tid=34626]]], state=connected] 2026-02-20T21:15:17.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:17.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:17.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=309, tag_tid=23, entry_tid=34629], [object_number=308, tag_tid=23, entry_tid=34628], [object_number=311, tag_tid=23, entry_tid=34627], [object_number=310, tag_tid=23, entry_tid=34626]]], state=connected] 2026-02-20T21:15:17.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=309, tag_tid=23, entry_tid=34629' 2026-02-20T21:15:17.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=309, tag_tid=23, entry_tid=34629' 2026-02-20T21:15:17.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=309, tag_tid=23, entry_tid=34629' '!=' 'object_number=308, tag_tid=23, entry_tid=34344' 2026-02-20T21:15:17.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=309, tag_tid=23, entry_tid=34629' = 'object_number=308, tag_tid=23, entry_tid=34344' 2026-02-20T21:15:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=309, tag_tid=23, entry_tid=34629' 2026-02-20T21:15:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:17.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:17.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:17.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=23, entry_tid=34629' 2026-02-20T21:15:17.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:17.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=23, entry_tid=34629' 2026-02-20T21:15:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:17.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34629 2026-02-20T21:15:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 34629 2026-02-20T21:15:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:17.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:17.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:17.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:17.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:17.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:17.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:17.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:17.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:17.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=308, tag_tid=23, entry_tid=34920], [object_number=311, tag_tid=23, entry_tid=34919], [object_number=310, tag_tid=23, entry_tid=34918], [object_number=309, tag_tid=23, entry_tid=34917]]], state=connected] 2026-02-20T21:15:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:17.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:17.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:17.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:17.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=308, tag_tid=23, entry_tid=34920], [object_number=311, tag_tid=23, entry_tid=34919], [object_number=310, tag_tid=23, entry_tid=34918], [object_number=309, tag_tid=23, entry_tid=34917]]], state=connected] 2026-02-20T21:15:17.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:17.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=308, tag_tid=23, entry_tid=34920' 2026-02-20T21:15:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=308, tag_tid=23, entry_tid=34920' 2026-02-20T21:15:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=308, tag_tid=23, entry_tid=34920' '!=' 'object_number=309, tag_tid=23, entry_tid=34629' 2026-02-20T21:15:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=308, tag_tid=23, entry_tid=34920' = 'object_number=309, tag_tid=23, entry_tid=34629' 2026-02-20T21:15:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=308, tag_tid=23, entry_tid=34920' 2026-02-20T21:15:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=308, tag_tid=23, entry_tid=34920' 2026-02-20T21:15:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:17.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:17.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:17.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:17.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:17.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=308, tag_tid=23, entry_tid=34920' 2026-02-20T21:15:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:17.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34920 2026-02-20T21:15:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 34920 2026-02-20T21:15:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:17.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:17.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:18.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:18.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:18.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=309, tag_tid=23, entry_tid=35201], [object_number=308, tag_tid=23, entry_tid=35200], [object_number=311, tag_tid=23, entry_tid=35199], [object_number=310, tag_tid=23, entry_tid=35198]]], state=connected] 2026-02-20T21:15:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=309, tag_tid=23, entry_tid=35201], [object_number=308, tag_tid=23, entry_tid=35200], [object_number=311, tag_tid=23, entry_tid=35199], [object_number=310, tag_tid=23, entry_tid=35198]]], state=connected] 2026-02-20T21:15:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=309, tag_tid=23, entry_tid=35201' 2026-02-20T21:15:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=309, tag_tid=23, entry_tid=35201' 2026-02-20T21:15:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=309, tag_tid=23, entry_tid=35201' '!=' 'object_number=308, tag_tid=23, entry_tid=34920' 2026-02-20T21:15:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:18.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=309, tag_tid=23, entry_tid=35201' = 'object_number=308, tag_tid=23, entry_tid=34920' 2026-02-20T21:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=309, tag_tid=23, entry_tid=35201' 2026-02-20T21:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:18.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=23, entry_tid=35201' 2026-02-20T21:15:18.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:18.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:18.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=23, entry_tid=35201' 2026-02-20T21:15:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35201 2026-02-20T21:15:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 35201 2026-02-20T21:15:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:18.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:18.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:18.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:18.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:18.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:18.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:18.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:18.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:18.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:18.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:18.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:18.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=309, tag_tid=23, entry_tid=35497], [object_number=308, tag_tid=23, entry_tid=35496], [object_number=311, tag_tid=23, entry_tid=35495], [object_number=310, tag_tid=23, entry_tid=35494]]], state=connected] 2026-02-20T21:15:18.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:18.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:18.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:18.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:18.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:18.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:18.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:18.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:18.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=309, tag_tid=23, entry_tid=35497], [object_number=308, tag_tid=23, entry_tid=35496], [object_number=311, tag_tid=23, entry_tid=35495], [object_number=310, tag_tid=23, entry_tid=35494]]], state=connected] 2026-02-20T21:15:18.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=309, tag_tid=23, entry_tid=35497' 2026-02-20T21:15:18.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=309, tag_tid=23, entry_tid=35497' 2026-02-20T21:15:18.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=309, tag_tid=23, entry_tid=35497' '!=' 'object_number=309, tag_tid=23, entry_tid=35201' 2026-02-20T21:15:18.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:18.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=309, tag_tid=23, entry_tid=35497' = 'object_number=309, tag_tid=23, entry_tid=35201' 2026-02-20T21:15:18.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=309, tag_tid=23, entry_tid=35497' 2026-02-20T21:15:18.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:18.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:18.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:18.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:18.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=23, entry_tid=35497' 2026-02-20T21:15:18.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:18.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:18.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:18.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:18.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:18.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:18.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:18.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=23, entry_tid=35497' 2026-02-20T21:15:18.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:18.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:18.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35497 2026-02-20T21:15:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 35497 2026-02-20T21:15:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:18.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:18.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:18.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:18.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:18.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:18.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:18.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:18.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:18.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:18.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:18.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:18.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=311, tag_tid=23, entry_tid=35823], [object_number=310, tag_tid=23, entry_tid=35822], [object_number=309, tag_tid=23, entry_tid=35821], [object_number=308, tag_tid=23, entry_tid=35820]]], state=connected] 2026-02-20T21:15:18.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:18.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:18.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:18.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:18.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:18.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:18.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:18.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:18.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:18.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:18.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:18.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:18.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:18.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:18.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:18.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:18.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:18.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=311, tag_tid=23, entry_tid=35823], [object_number=310, tag_tid=23, entry_tid=35822], [object_number=309, tag_tid=23, entry_tid=35821], [object_number=308, tag_tid=23, entry_tid=35820]]], state=connected] 2026-02-20T21:15:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=311, tag_tid=23, entry_tid=35823' 2026-02-20T21:15:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=311, tag_tid=23, entry_tid=35823' 2026-02-20T21:15:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=311, tag_tid=23, entry_tid=35823' '!=' 'object_number=309, tag_tid=23, entry_tid=35497' 2026-02-20T21:15:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=311, tag_tid=23, entry_tid=35823' = 'object_number=309, tag_tid=23, entry_tid=35497' 2026-02-20T21:15:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=311, tag_tid=23, entry_tid=35823' 2026-02-20T21:15:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:18.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=311, tag_tid=23, entry_tid=35823' 2026-02-20T21:15:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:18.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:18.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:18.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:18.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:18.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:18.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=311, tag_tid=23, entry_tid=35823' 2026-02-20T21:15:18.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:18.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:18.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35823 2026-02-20T21:15:18.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 35823 2026-02-20T21:15:18.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:18.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:18.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:18.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:18.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:18.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:19.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:19.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:19.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:19.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=311, tag_tid=23, entry_tid=36083], [object_number=310, tag_tid=23, entry_tid=36082], [object_number=309, tag_tid=23, entry_tid=36081], [object_number=308, tag_tid=23, entry_tid=36080]]], state=connected] 2026-02-20T21:15:19.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:19.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:19.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:19.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=311, tag_tid=23, entry_tid=36083], [object_number=310, tag_tid=23, entry_tid=36082], [object_number=309, tag_tid=23, entry_tid=36081], [object_number=308, tag_tid=23, entry_tid=36080]]], state=connected] 2026-02-20T21:15:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=311, tag_tid=23, entry_tid=36083' 2026-02-20T21:15:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=311, tag_tid=23, entry_tid=36083' 2026-02-20T21:15:19.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=311, tag_tid=23, entry_tid=36083' '!=' 'object_number=311, tag_tid=23, entry_tid=35823' 2026-02-20T21:15:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=311, tag_tid=23, entry_tid=36083' = 'object_number=311, tag_tid=23, entry_tid=35823' 2026-02-20T21:15:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=311, tag_tid=23, entry_tid=36083' 2026-02-20T21:15:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:19.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=311, tag_tid=23, entry_tid=36083' 2026-02-20T21:15:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=311, tag_tid=23, entry_tid=36083' 2026-02-20T21:15:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:19.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36083 2026-02-20T21:15:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 36083 2026-02-20T21:15:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:19.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:19.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:19.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:19.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:19.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:19.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:19.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:19.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=311, tag_tid=23, entry_tid=36411], [object_number=310, tag_tid=23, entry_tid=36410], [object_number=309, tag_tid=23, entry_tid=36409], [object_number=308, tag_tid=23, entry_tid=36408]]], state=connected] 2026-02-20T21:15:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:19.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:19.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:19.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:19.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:19.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:19.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:19.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=311, tag_tid=23, entry_tid=36411], [object_number=310, tag_tid=23, entry_tid=36410], [object_number=309, tag_tid=23, entry_tid=36409], [object_number=308, tag_tid=23, entry_tid=36408]]], state=connected] 2026-02-20T21:15:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=311, tag_tid=23, entry_tid=36411' 2026-02-20T21:15:19.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=311, tag_tid=23, entry_tid=36411' 2026-02-20T21:15:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=311, tag_tid=23, entry_tid=36411' '!=' 'object_number=311, tag_tid=23, entry_tid=36083' 2026-02-20T21:15:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=311, tag_tid=23, entry_tid=36411' = 'object_number=311, tag_tid=23, entry_tid=36083' 2026-02-20T21:15:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=311, tag_tid=23, entry_tid=36411' 2026-02-20T21:15:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=311, tag_tid=23, entry_tid=36411' 2026-02-20T21:15:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:19.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:19.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:19.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:19.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:19.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=311, tag_tid=23, entry_tid=36411' 2026-02-20T21:15:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36411 2026-02-20T21:15:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 36411 2026-02-20T21:15:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:19.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:19.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:19.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:19.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:19.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:19.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:19.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:19.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:19.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:19.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:19.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:19.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:19.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:19.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:19.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=310, tag_tid=23, entry_tid=36446], [object_number=309, tag_tid=23, entry_tid=36445], [object_number=308, tag_tid=23, entry_tid=36444], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:19.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:19.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:19.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:19.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:19.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:19.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:19.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:19.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:19.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:19.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:19.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:19.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=310, tag_tid=23, entry_tid=36446], [object_number=309, tag_tid=23, entry_tid=36445], [object_number=308, tag_tid=23, entry_tid=36444], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:19.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=310, tag_tid=23, entry_tid=36446' '!=' 'object_number=311, tag_tid=23, entry_tid=36411' 2026-02-20T21:15:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:15:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=310, tag_tid=23, entry_tid=36446' = 'object_number=311, tag_tid=23, entry_tid=36411' 2026-02-20T21:15:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:19.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:19.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=24 2026-02-20T21:15:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:15:19.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:19.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=23 2026-02-20T21:15:19.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:19.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:19.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:19.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:15:19.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:19.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:15:19.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:15:19.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36446 2026-02-20T21:15:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 24 = 23 -a 1 -le 36446 2026-02-20T21:15:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:15:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:19.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:19.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:19.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:19.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:19.997 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7601952882815167) > weight (0.25) 2026-02-20T21:15:19.997 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-20T21:15:19.997 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-20T21:15:19.998 INFO:tasks.rbd_mirror_thrash:waiting for 0.8156785609787753 secs before reviving daemons 2026-02-20T21:15:19.998 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:15:19.996+0000 7fc3e0bcf640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 60128) UID: 1000 2026-02-20T21:15:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:20.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:20.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:20.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:20.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:20.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:20.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:20.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:20.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:20.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:20.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:20.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:20.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:20.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:20.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:20.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:20.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:20.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:20.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:20.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=310, tag_tid=23, entry_tid=36446], [object_number=309, tag_tid=23, entry_tid=36445], [object_number=308, tag_tid=23, entry_tid=36444], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:20.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:20.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:20.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:20.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:20.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:20.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:20.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:20.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:20.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=310, tag_tid=23, entry_tid=36446], [object_number=309, tag_tid=23, entry_tid=36445], [object_number=308, tag_tid=23, entry_tid=36444], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:20.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=310, tag_tid=23, entry_tid=36446' '!=' 'object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:20.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:15:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:20.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:20.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:20.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:20.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:20.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:20.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:20.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=310, tag_tid=23, entry_tid=36446], [object_number=309, tag_tid=23, entry_tid=36445], [object_number=308, tag_tid=23, entry_tid=36444], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:20.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:20.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:20.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:20.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:20.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:20.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:20.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=310, tag_tid=23, entry_tid=36446], [object_number=309, tag_tid=23, entry_tid=36445], [object_number=308, tag_tid=23, entry_tid=36444], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=310, tag_tid=23, entry_tid=36446' '!=' 'object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:15:20.814 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-20T21:15:20.814 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:15:20.815 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-20T21:15:20.815 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-20T21:15:20.815 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-20T21:15:20.817 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-20T21:15:20.817 INFO:tasks.rbd_mirror_thrash:waiting for 104.16050558419218 secs before thrashing 2026-02-20T21:15:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:21.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:21.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:21.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:21.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:21.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:21.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:21.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:21.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:21.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:21.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=310, tag_tid=23, entry_tid=36446], [object_number=309, tag_tid=23, entry_tid=36445], [object_number=308, tag_tid=23, entry_tid=36444], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:21.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:21.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:21.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:21.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:21.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:21.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:21.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:21.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:21.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:21.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=310, tag_tid=23, entry_tid=36446], [object_number=309, tag_tid=23, entry_tid=36445], [object_number=308, tag_tid=23, entry_tid=36444], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:21.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:21.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:21.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:21.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=310, tag_tid=23, entry_tid=36446' '!=' 'object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:21.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:21.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:15:23.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:23.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:23.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:23.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:23.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:23.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:23.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:23.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:23.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:23.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=310, tag_tid=23, entry_tid=36446], [object_number=309, tag_tid=23, entry_tid=36445], [object_number=308, tag_tid=23, entry_tid=36444], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:23.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:23.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:23.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:23.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:23.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:23.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:23.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:23.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:23.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:23.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=310, tag_tid=23, entry_tid=36446], [object_number=309, tag_tid=23, entry_tid=36445], [object_number=308, tag_tid=23, entry_tid=36444], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:23.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:23.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=310, tag_tid=23, entry_tid=36446' '!=' 'object_number=310, tag_tid=23, entry_tid=36446' 2026-02-20T21:15:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:15:23.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:15:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:15:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:15:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:15:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:15:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:15:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:15:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:15:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:15:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:15:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:15:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:15:25.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:15:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:15:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:15:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:15:25.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:15:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:15:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:15:25.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:25.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:25.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:25.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:25.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:25.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:25.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:25.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:25.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:25.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:25.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:15:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:15:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:15:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:15:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:15:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:15:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:15:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:15:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:15:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:15:25.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:25.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 77 2026-02-20T21:15:25.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 77 2026-02-20T21:15:25.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:15:25.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:25.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=309, tag_tid=24, entry_tid=1], [object_number=308, tag_tid=24, entry_tid=0], [object_number=310, tag_tid=23, entry_tid=36446], [object_number=311, tag_tid=23, entry_tid=36443]]], state=connected] 2026-02-20T21:15:25.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:15:25.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:25.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=309, tag_tid=24, entry_tid=1' -a 'object_number=309, tag_tid=24, entry_tid=1' = 'object_number=309, tag_tid=24, entry_tid=1' 2026-02-20T21:15:25.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap8 2026-02-20T21:15:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8 2026-02-20T21:15:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:15:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:15:25.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:15:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap8 2026-02-20T21:15:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-20T21:15:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:15:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:15:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 15 2026-02-20T21:15:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ba958b175d 2026-02-20T21:15:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ba958b175d 2026-02-20T21:15:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:15:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:15:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:15:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:15:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:06:40 2026 2026-02-20T21:15:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:14:31 2026 2026-02-20T21:15:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:14:39 2026 2026-02-20T21:15:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:15:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ba958b175d 2026-02-20T21:15:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:15:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:15:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:15:26.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:15:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap7-clean ']' 2026-02-20T21:15:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap7-clean 2026-02-20T21:15:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7-clean 2026-02-20T21:15:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:15:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:15:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:15:26.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:15:26.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:15:27.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:15:27.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:15:27.121 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.115+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:27.121 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.115+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:27.121 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.116+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:27.121 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.116+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:27.121 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.117+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:27.121 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.117+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:27.214 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.211+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:15:27.215 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.212+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:15:27.215 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.212+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:15:27.248 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.245+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:15:27.248 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.245+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:15:27.249 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.246+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:27.284 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.281+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:15:27.285 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.282+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:15:27.287 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.284+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:27.287 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.284+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:27.287 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.284+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:27.287 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.284+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:27.323 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.321+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:27.324 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.321+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:27.325 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.322+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:27.325 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.322+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:27.325 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.322+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:27.395 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.393+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:15:27.396 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.393+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:15:27.396 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:27.393+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:27.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:15:27.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:15:27.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:15:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:27.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap8 2026-02-20T21:15:27.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:27.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:27.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8 2026-02-20T21:15:27.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:15:27.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:15:27.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:15:27.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:15:27.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:15:28.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:15:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:15:28.258 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.256+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:15:28.260 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.257+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:28.263 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.260+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:15:28.263 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.261+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:15:28.264 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.261+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:15:28.264 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.261+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:15:28.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.420+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:28.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.422+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:28.425 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.422+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:28.427 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.424+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:15:28.427 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.425+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:15:28.428 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.425+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:15:28.498 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.496+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:15:28.499 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.496+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:15:28.499 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5605873878285188) > weight (0.25) 2026-02-20T21:15:28.499 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-20T21:15:28.500 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-20T21:15:28.500 INFO:tasks.rbd_mirror_thrash:waiting for 6.068957550098418 secs before reviving daemons 2026-02-20T21:15:28.500 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.497+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:28.501 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.497+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:28.501 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.497+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:15:28.501 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:15:28.499+0000 7fe0e98c6640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 65534) UID: 1000 2026-02-20T21:15:28.532 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.529+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:15:28.532 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.529+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:28.532 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.529+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:28.533 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.530+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:28.533 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.530+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:28.533 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.530+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:28.570 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.568+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:15:28.571 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.568+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:28.571 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.568+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:28.571 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:28.568+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:15:28.638 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:15:28.636+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:15:28.639 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:15:28.636+0000 7f412fcd2640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:15:28.640 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:15:28.637+0000 7f412ecd0640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:15:28.640 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:15:28.637+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:15:28.641 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:15:28.638+0000 7f412c4cb640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:15:28.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:15:28.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:15:28.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:15:28.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap8-clean 2026-02-20T21:15:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap8-clean 2026-02-20T21:15:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:15:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8-clean 2026-02-20T21:15:28.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap8-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:15:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:15:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:15:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:15:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:29.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:15:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-20T21:15:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:15:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:15:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 67913728 bytes 2026-02-20T21:15:31.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 111625216 bytes 2026-02-20T21:15:32.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 152537088 bytes 2026-02-20T21:15:32.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 185111552 bytes 2026-02-20T21:15:32.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 197317632 bytes 2026-02-20T21:15:33.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 275052544 bytes 2026-02-20T21:15:34.569 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-20T21:15:34.569 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:15:34.570 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-20T21:15:34.570 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-20T21:15:34.570 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-20T21:15:34.573 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-20T21:15:34.573 INFO:tasks.rbd_mirror_thrash:waiting for 90.48344446128547 secs before thrashing 2026-02-20T21:15:34.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 356144128 bytes 2026-02-20T21:15:34.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 359005184 bytes 2026-02-20T21:15:34.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 365125632 bytes 2026-02-20T21:15:35.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 391730176 bytes 2026-02-20T21:15:35.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 392328192 bytes 2026-02-20T21:15:36.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 414850048 bytes 2026-02-20T21:15:36.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 443858944 bytes 2026-02-20T21:15:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 513486848 bytes 2026-02-20T21:15:38.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 574936064 bytes 2026-02-20T21:15:38.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 637394944 bytes 2026-02-20T21:15:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 673197056 bytes 2026-02-20T21:15:40.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 764265472 bytes 2026-02-20T21:15:41.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 782636032 bytes 2026-02-20T21:15:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 868795392 bytes 2026-02-20T21:15:42.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 907483136 bytes 2026-02-20T21:15:43.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2200 IOs, 950897664 bytes 2026-02-20T21:15:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:15:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:15:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:15:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:15:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:15:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:15:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:15:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:15:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:15:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:15:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:15:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:15:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:15:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:15:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:15:44.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:15:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:15:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:15:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-20T21:15:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":8301549.47,"entries_behind_primary":20751,"entries_per_second":517.73,"non_primary_position":{"entry_tid":1553,"object_number":309,"tag_tid":26},"primary_position":{"entry_tid":22304,"object_number":328,"tag_tid":26},"seconds_until_synced":40} 2026-02-20T21:15:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.0 on trial163 2026-02-20T21:15:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:15:38 2026-02-20T21:15:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:15:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:15:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:15:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:15:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:15:38' 2026-02-20T21:15:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:15:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-20T21:15:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-20T21:15:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":8301549.47,"entries_behind_primary":20751,"entries_per_second":517.73,"non_primary_position":{"entry_tid":1553,"object_number":309,"tag_tid":26},"primary_position":{"entry_tid":22304,"object_number":328,"tag_tid":26},"seconds_until_synced":40} 2026-02-20T21:15:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:15:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:15:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-20T21:15:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:15:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:15:44.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.0 on trial163 2026-02-20T21:15:44.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-20T21:15:44.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:15:44.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:15:38' 2026-02-20T21:15:44.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:15:38' 2026-02-20T21:15:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:15:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:15:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-20 21:15:38' 2026-02-20T21:15:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:15:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:15:44.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:15:44.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142044287mirror.0trial16320.2.0-677-gf7870454trueOK43304332mirror.2trial16320.2.0-677-gf7870454falseOK43484351mirror.3trial16320.2.0-677-gf7870454falseOK45004503mirror.1trial16320.2.0-677-gf7870454falseOKtestba923525-490d-4a76-8e5f-b742509a94daup+replayingreplaying, {"bytes_per_second":8301549.47,"entries_behind_primary":20751,"entries_per_second":517.73,"non_primary_position":{"entry_tid":1553,"object_number":309,"tag_tid":26},"primary_position":{"entry_tid":22304,"object_number":328,"tag_tid":26},"seconds_until_synced":40}42044287mirror.0trial1632026-02-20 21:15:38cluster28b968eaf-6c24-469f-bea5-c8acb780219eup+stoppedlocal image is primary2026-02-20 21:15:38' 2026-02-20T21:15:44.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-20T21:15:44.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:15:44.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:15:38' 2026-02-20T21:15:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:15:44.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-20T21:15:44.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:15:44.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-20T21:15:44.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:15:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:15:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap9 2026-02-20T21:15:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap9 2026-02-20T21:15:44.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:15:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:15:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:15:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9 2026-02-20T21:15:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:15:49.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:49.243+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f8820 2026-02-20T21:15:49.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:49.243+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440340 2026-02-20T21:15:49.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:49.243+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe65b0 2026-02-20T21:15:49.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:15:49.244+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:16:15.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:16:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:16:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:16:15.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:16:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:16:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:16:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:16:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:16:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:16:16.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:16:16.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:16:16.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:16:16.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:16:16.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:16.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:16:16.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:16:16.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:16.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:16:16.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:16.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:16:16.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:16.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:16:16.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:16:16.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:16.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:16.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:16.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:16.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:16:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:16:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:16:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:16:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:16:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:16:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:16:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:16:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:16:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:16:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:16:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:16:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:16:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:16:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:16:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:16:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:16:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:16:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:16:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:16:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:16:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:17.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:17.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:17.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:17.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:17.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:17.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:17.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:17.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=349, tag_tid=26, entry_tid=39741], [object_number=348, tag_tid=26, entry_tid=39740], [object_number=351, tag_tid=26, entry_tid=39739], [object_number=350, tag_tid=26, entry_tid=39738]]], state=connected] 2026-02-20T21:16:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:17.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:17.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:17.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=349, tag_tid=26, entry_tid=39741], [object_number=348, tag_tid=26, entry_tid=39740], [object_number=351, tag_tid=26, entry_tid=39739], [object_number=350, tag_tid=26, entry_tid=39738]]], state=connected] 2026-02-20T21:16:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=349, tag_tid=26, entry_tid=39741' 2026-02-20T21:16:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=349, tag_tid=26, entry_tid=39741' 2026-02-20T21:16:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=349, tag_tid=26, entry_tid=39741' '!=' '' 2026-02-20T21:16:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=349, tag_tid=26, entry_tid=39741' = '' 2026-02-20T21:16:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=349, tag_tid=26, entry_tid=39741' 2026-02-20T21:16:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=349, tag_tid=26, entry_tid=39741' 2026-02-20T21:16:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:17.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:17.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=349, tag_tid=26, entry_tid=39741' 2026-02-20T21:16:17.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:17.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39741 2026-02-20T21:16:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 39741 2026-02-20T21:16:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:17.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:17.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:17.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:17.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:17.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:17.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:17.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:17.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=348, tag_tid=26, entry_tid=40028], [object_number=351, tag_tid=26, entry_tid=40027], [object_number=350, tag_tid=26, entry_tid=40026], [object_number=349, tag_tid=26, entry_tid=40025]]], state=connected] 2026-02-20T21:16:17.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:17.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:17.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:17.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:17.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=348, tag_tid=26, entry_tid=40028], [object_number=351, tag_tid=26, entry_tid=40027], [object_number=350, tag_tid=26, entry_tid=40026], [object_number=349, tag_tid=26, entry_tid=40025]]], state=connected] 2026-02-20T21:16:17.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=348, tag_tid=26, entry_tid=40028' 2026-02-20T21:16:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=348, tag_tid=26, entry_tid=40028' 2026-02-20T21:16:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=348, tag_tid=26, entry_tid=40028' '!=' 'object_number=349, tag_tid=26, entry_tid=39741' 2026-02-20T21:16:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=348, tag_tid=26, entry_tid=40028' = 'object_number=349, tag_tid=26, entry_tid=39741' 2026-02-20T21:16:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=348, tag_tid=26, entry_tid=40028' 2026-02-20T21:16:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:17.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:17.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=348, tag_tid=26, entry_tid=40028' 2026-02-20T21:16:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=348, tag_tid=26, entry_tid=40028' 2026-02-20T21:16:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40028 2026-02-20T21:16:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 40028 2026-02-20T21:16:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:17.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:17.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:17.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:17.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:17.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:17.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:17.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:17.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=350, tag_tid=26, entry_tid=40342], [object_number=349, tag_tid=26, entry_tid=40341], [object_number=348, tag_tid=26, entry_tid=40340], [object_number=351, tag_tid=26, entry_tid=40339]]], state=connected] 2026-02-20T21:16:17.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:17.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:17.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:17.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:17.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:17.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:17.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:17.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=350, tag_tid=26, entry_tid=40342], [object_number=349, tag_tid=26, entry_tid=40341], [object_number=348, tag_tid=26, entry_tid=40340], [object_number=351, tag_tid=26, entry_tid=40339]]], state=connected] 2026-02-20T21:16:17.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:17.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=350, tag_tid=26, entry_tid=40342' 2026-02-20T21:16:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=350, tag_tid=26, entry_tid=40342' 2026-02-20T21:16:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=350, tag_tid=26, entry_tid=40342' '!=' 'object_number=348, tag_tid=26, entry_tid=40028' 2026-02-20T21:16:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=350, tag_tid=26, entry_tid=40342' = 'object_number=348, tag_tid=26, entry_tid=40028' 2026-02-20T21:16:17.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=350, tag_tid=26, entry_tid=40342' 2026-02-20T21:16:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:17.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=350, tag_tid=26, entry_tid=40342' 2026-02-20T21:16:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:17.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:17.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=350, tag_tid=26, entry_tid=40342' 2026-02-20T21:16:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:17.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40342 2026-02-20T21:16:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 40342 2026-02-20T21:16:17.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:18.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:18.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:18.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:18.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:18.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:18.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:18.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:18.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:18.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:18.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:18.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:18.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:18.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:18.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=348, tag_tid=26, entry_tid=40668], [object_number=351, tag_tid=26, entry_tid=40667], [object_number=350, tag_tid=26, entry_tid=40666], [object_number=349, tag_tid=26, entry_tid=40665]]], state=connected] 2026-02-20T21:16:18.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:18.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:18.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:18.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=348, tag_tid=26, entry_tid=40668], [object_number=351, tag_tid=26, entry_tid=40667], [object_number=350, tag_tid=26, entry_tid=40666], [object_number=349, tag_tid=26, entry_tid=40665]]], state=connected] 2026-02-20T21:16:18.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=348, tag_tid=26, entry_tid=40668' 2026-02-20T21:16:18.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=348, tag_tid=26, entry_tid=40668' 2026-02-20T21:16:18.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=348, tag_tid=26, entry_tid=40668' '!=' 'object_number=350, tag_tid=26, entry_tid=40342' 2026-02-20T21:16:18.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:18.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=348, tag_tid=26, entry_tid=40668' = 'object_number=350, tag_tid=26, entry_tid=40342' 2026-02-20T21:16:18.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=348, tag_tid=26, entry_tid=40668' 2026-02-20T21:16:18.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:18.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:18.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:18.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:18.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=348, tag_tid=26, entry_tid=40668' 2026-02-20T21:16:18.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:18.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:18.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:18.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:18.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=348, tag_tid=26, entry_tid=40668' 2026-02-20T21:16:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:18.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:18.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40668 2026-02-20T21:16:18.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 40668 2026-02-20T21:16:18.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:18.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:18.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:18.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:18.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:18.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:18.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:18.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:18.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:18.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:18.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:18.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:18.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:18.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:18.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:18.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:18.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:18.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:18.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:18.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=351, tag_tid=26, entry_tid=40959], [object_number=350, tag_tid=26, entry_tid=40958], [object_number=349, tag_tid=26, entry_tid=40957], [object_number=348, tag_tid=26, entry_tid=40956]]], state=connected] 2026-02-20T21:16:18.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:18.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:18.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:18.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=351, tag_tid=26, entry_tid=40959], [object_number=350, tag_tid=26, entry_tid=40958], [object_number=349, tag_tid=26, entry_tid=40957], [object_number=348, tag_tid=26, entry_tid=40956]]], state=connected] 2026-02-20T21:16:18.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=351, tag_tid=26, entry_tid=40959' 2026-02-20T21:16:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=351, tag_tid=26, entry_tid=40959' 2026-02-20T21:16:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=351, tag_tid=26, entry_tid=40959' '!=' 'object_number=348, tag_tid=26, entry_tid=40668' 2026-02-20T21:16:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=351, tag_tid=26, entry_tid=40959' = 'object_number=348, tag_tid=26, entry_tid=40668' 2026-02-20T21:16:18.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=351, tag_tid=26, entry_tid=40959' 2026-02-20T21:16:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:18.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:18.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:18.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=351, tag_tid=26, entry_tid=40959' 2026-02-20T21:16:18.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:18.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:18.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:18.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:18.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:18.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=351, tag_tid=26, entry_tid=40959' 2026-02-20T21:16:18.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:18.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:18.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40959 2026-02-20T21:16:18.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 40959 2026-02-20T21:16:18.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:18.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:18.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:18.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:18.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:18.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:18.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:18.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:18.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:18.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:18.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:18.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:18.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=348, tag_tid=26, entry_tid=41272], [object_number=351, tag_tid=26, entry_tid=41271], [object_number=350, tag_tid=26, entry_tid=41270], [object_number=349, tag_tid=26, entry_tid=41269]]], state=connected] 2026-02-20T21:16:18.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:18.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:18.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:18.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:18.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:18.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:18.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:18.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:18.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:18.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:18.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:18.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:18.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:18.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:18.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=348, tag_tid=26, entry_tid=41272], [object_number=351, tag_tid=26, entry_tid=41271], [object_number=350, tag_tid=26, entry_tid=41270], [object_number=349, tag_tid=26, entry_tid=41269]]], state=connected] 2026-02-20T21:16:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=348, tag_tid=26, entry_tid=41272' 2026-02-20T21:16:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=348, tag_tid=26, entry_tid=41272' 2026-02-20T21:16:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=348, tag_tid=26, entry_tid=41272' '!=' 'object_number=351, tag_tid=26, entry_tid=40959' 2026-02-20T21:16:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=348, tag_tid=26, entry_tid=41272' = 'object_number=351, tag_tid=26, entry_tid=40959' 2026-02-20T21:16:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=348, tag_tid=26, entry_tid=41272' 2026-02-20T21:16:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:18.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:18.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=348, tag_tid=26, entry_tid=41272' 2026-02-20T21:16:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:18.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:18.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:18.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:18.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:18.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=348, tag_tid=26, entry_tid=41272' 2026-02-20T21:16:18.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:18.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41272 2026-02-20T21:16:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 41272 2026-02-20T21:16:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:18.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:19.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:19.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:19.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:19.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:19.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:19.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:19.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:19.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:19.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:19.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:19.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=350, tag_tid=26, entry_tid=41574], [object_number=349, tag_tid=26, entry_tid=41573], [object_number=348, tag_tid=26, entry_tid=41572], [object_number=351, tag_tid=26, entry_tid=41571]]], state=connected] 2026-02-20T21:16:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:19.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:19.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:19.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:19.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:19.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:19.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=350, tag_tid=26, entry_tid=41574], [object_number=349, tag_tid=26, entry_tid=41573], [object_number=348, tag_tid=26, entry_tid=41572], [object_number=351, tag_tid=26, entry_tid=41571]]], state=connected] 2026-02-20T21:16:19.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=350, tag_tid=26, entry_tid=41574' 2026-02-20T21:16:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=350, tag_tid=26, entry_tid=41574' 2026-02-20T21:16:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=350, tag_tid=26, entry_tid=41574' '!=' 'object_number=348, tag_tid=26, entry_tid=41272' 2026-02-20T21:16:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=350, tag_tid=26, entry_tid=41574' = 'object_number=348, tag_tid=26, entry_tid=41272' 2026-02-20T21:16:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=350, tag_tid=26, entry_tid=41574' 2026-02-20T21:16:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:19.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:19.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=350, tag_tid=26, entry_tid=41574' 2026-02-20T21:16:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:19.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:19.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:19.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:19.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=350, tag_tid=26, entry_tid=41574' 2026-02-20T21:16:19.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:19.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41574 2026-02-20T21:16:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 41574 2026-02-20T21:16:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:19.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:19.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:19.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:19.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:19.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:19.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:19.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:19.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:19.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:19.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:19.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:19.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:19.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=349, tag_tid=26, entry_tid=42069], [object_number=348, tag_tid=26, entry_tid=42068], [object_number=351, tag_tid=26, entry_tid=42067], [object_number=350, tag_tid=26, entry_tid=42066]]], state=connected] 2026-02-20T21:16:19.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:19.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:19.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:19.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:19.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:19.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:19.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:19.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:19.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:19.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=349, tag_tid=26, entry_tid=42069], [object_number=348, tag_tid=26, entry_tid=42068], [object_number=351, tag_tid=26, entry_tid=42067], [object_number=350, tag_tid=26, entry_tid=42066]]], state=connected] 2026-02-20T21:16:19.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:19.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=349, tag_tid=26, entry_tid=42069' 2026-02-20T21:16:19.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=349, tag_tid=26, entry_tid=42069' 2026-02-20T21:16:19.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=349, tag_tid=26, entry_tid=42069' '!=' 'object_number=350, tag_tid=26, entry_tid=41574' 2026-02-20T21:16:19.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:19.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=349, tag_tid=26, entry_tid=42069' = 'object_number=350, tag_tid=26, entry_tid=41574' 2026-02-20T21:16:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=349, tag_tid=26, entry_tid=42069' 2026-02-20T21:16:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:19.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=349, tag_tid=26, entry_tid=42069' 2026-02-20T21:16:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:19.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:19.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:19.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=349, tag_tid=26, entry_tid=42069' 2026-02-20T21:16:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42069 2026-02-20T21:16:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 42069 2026-02-20T21:16:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:19.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:19.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:19.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:19.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:19.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:19.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:19.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:20.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:20.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:20.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:20.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:20.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=351, tag_tid=26, entry_tid=42379], [object_number=350, tag_tid=26, entry_tid=42378], [object_number=349, tag_tid=26, entry_tid=42377], [object_number=348, tag_tid=26, entry_tid=42376]]], state=connected] 2026-02-20T21:16:20.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:20.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:20.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:20.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:20.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:20.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:20.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:20.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=351, tag_tid=26, entry_tid=42379], [object_number=350, tag_tid=26, entry_tid=42378], [object_number=349, tag_tid=26, entry_tid=42377], [object_number=348, tag_tid=26, entry_tid=42376]]], state=connected] 2026-02-20T21:16:20.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=351, tag_tid=26, entry_tid=42379' 2026-02-20T21:16:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=351, tag_tid=26, entry_tid=42379' 2026-02-20T21:16:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=351, tag_tid=26, entry_tid=42379' '!=' 'object_number=349, tag_tid=26, entry_tid=42069' 2026-02-20T21:16:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=351, tag_tid=26, entry_tid=42379' = 'object_number=349, tag_tid=26, entry_tid=42069' 2026-02-20T21:16:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=351, tag_tid=26, entry_tid=42379' 2026-02-20T21:16:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:20.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:20.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:20.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:20.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=351, tag_tid=26, entry_tid=42379' 2026-02-20T21:16:20.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:20.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=351, tag_tid=26, entry_tid=42379' 2026-02-20T21:16:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:20.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42379 2026-02-20T21:16:20.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 42379 2026-02-20T21:16:20.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:20.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:20.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:20.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:20.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:20.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:20.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:20.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:20.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:20.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:20.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:20.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:20.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:20.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:20.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:20.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:20.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=348, tag_tid=26, entry_tid=42716], [object_number=351, tag_tid=26, entry_tid=42715], [object_number=350, tag_tid=26, entry_tid=42714], [object_number=349, tag_tid=26, entry_tid=42713]]], state=connected] 2026-02-20T21:16:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=348, tag_tid=26, entry_tid=42716], [object_number=351, tag_tid=26, entry_tid=42715], [object_number=350, tag_tid=26, entry_tid=42714], [object_number=349, tag_tid=26, entry_tid=42713]]], state=connected] 2026-02-20T21:16:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=348, tag_tid=26, entry_tid=42716' 2026-02-20T21:16:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=348, tag_tid=26, entry_tid=42716' 2026-02-20T21:16:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=348, tag_tid=26, entry_tid=42716' '!=' 'object_number=351, tag_tid=26, entry_tid=42379' 2026-02-20T21:16:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=348, tag_tid=26, entry_tid=42716' = 'object_number=351, tag_tid=26, entry_tid=42379' 2026-02-20T21:16:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=348, tag_tid=26, entry_tid=42716' 2026-02-20T21:16:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=348, tag_tid=26, entry_tid=42716' 2026-02-20T21:16:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:20.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=348, tag_tid=26, entry_tid=42716' 2026-02-20T21:16:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:20.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42716 2026-02-20T21:16:20.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 42716 2026-02-20T21:16:20.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:20.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:20.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:20.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:20.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:20.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:20.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:20.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:20.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:20.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:20.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:20.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:20.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:20.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:20.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:20.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:20.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:20.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:20.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:20.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:20.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:20.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:20.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:20.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:20.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:20.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:20.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=348, tag_tid=26, entry_tid=43052], [object_number=351, tag_tid=26, entry_tid=43051], [object_number=350, tag_tid=26, entry_tid=43050], [object_number=349, tag_tid=26, entry_tid=43049]]], state=connected] 2026-02-20T21:16:20.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:20.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:20.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 87 2026-02-20T21:16:20.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:20.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:20.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:20.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=348, tag_tid=26, entry_tid=43052], [object_number=351, tag_tid=26, entry_tid=43051], [object_number=350, tag_tid=26, entry_tid=43050], [object_number=349, tag_tid=26, entry_tid=43049]]], state=connected] 2026-02-20T21:16:20.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=348, tag_tid=26, entry_tid=43052' 2026-02-20T21:16:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=348, tag_tid=26, entry_tid=43052' 2026-02-20T21:16:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=348, tag_tid=26, entry_tid=43052' '!=' 'object_number=348, tag_tid=26, entry_tid=42716' 2026-02-20T21:16:20.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=348, tag_tid=26, entry_tid=43052' = 'object_number=348, tag_tid=26, entry_tid=42716' 2026-02-20T21:16:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=348, tag_tid=26, entry_tid=43052' 2026-02-20T21:16:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:21.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:21.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=348, tag_tid=26, entry_tid=43052' 2026-02-20T21:16:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=348, tag_tid=26, entry_tid=43052' 2026-02-20T21:16:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:21.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43052 2026-02-20T21:16:21.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 43052 2026-02-20T21:16:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:21.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:21.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:21.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:21.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:21.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:21.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:21.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:21.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:21.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:21.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:21.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:21.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:21.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:21.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:21.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:21.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:21.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:21.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:21.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=354, tag_tid=26, entry_tid=43486], [object_number=353, tag_tid=26, entry_tid=43485], [object_number=352, tag_tid=26, entry_tid=43484], [object_number=355, tag_tid=26, entry_tid=43483]]], state=connected] 2026-02-20T21:16:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:21.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:21.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:21.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=354, tag_tid=26, entry_tid=43486], [object_number=353, tag_tid=26, entry_tid=43485], [object_number=352, tag_tid=26, entry_tid=43484], [object_number=355, tag_tid=26, entry_tid=43483]]], state=connected] 2026-02-20T21:16:21.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=354, tag_tid=26, entry_tid=43486' 2026-02-20T21:16:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=354, tag_tid=26, entry_tid=43486' 2026-02-20T21:16:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=354, tag_tid=26, entry_tid=43486' '!=' 'object_number=348, tag_tid=26, entry_tid=43052' 2026-02-20T21:16:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=354, tag_tid=26, entry_tid=43486' = 'object_number=348, tag_tid=26, entry_tid=43052' 2026-02-20T21:16:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=354, tag_tid=26, entry_tid=43486' 2026-02-20T21:16:21.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:21.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:21.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:21.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=354, tag_tid=26, entry_tid=43486' 2026-02-20T21:16:21.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:21.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:21.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:21.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:21.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:21.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:21.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=354, tag_tid=26, entry_tid=43486' 2026-02-20T21:16:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:21.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43486 2026-02-20T21:16:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 43486 2026-02-20T21:16:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:21.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:21.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:21.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:21.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:21.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:21.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:21.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:21.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=353, tag_tid=26, entry_tid=43805], [object_number=352, tag_tid=26, entry_tid=43804], [object_number=355, tag_tid=26, entry_tid=43803], [object_number=354, tag_tid=26, entry_tid=43802]]], state=connected] 2026-02-20T21:16:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:21.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=353, tag_tid=26, entry_tid=43805], [object_number=352, tag_tid=26, entry_tid=43804], [object_number=355, tag_tid=26, entry_tid=43803], [object_number=354, tag_tid=26, entry_tid=43802]]], state=connected] 2026-02-20T21:16:21.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=353, tag_tid=26, entry_tid=43805' 2026-02-20T21:16:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=353, tag_tid=26, entry_tid=43805' 2026-02-20T21:16:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=353, tag_tid=26, entry_tid=43805' '!=' 'object_number=354, tag_tid=26, entry_tid=43486' 2026-02-20T21:16:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=353, tag_tid=26, entry_tid=43805' = 'object_number=354, tag_tid=26, entry_tid=43486' 2026-02-20T21:16:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=353, tag_tid=26, entry_tid=43805' 2026-02-20T21:16:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=353, tag_tid=26, entry_tid=43805' 2026-02-20T21:16:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=353, tag_tid=26, entry_tid=43805' 2026-02-20T21:16:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:21.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=43805 2026-02-20T21:16:21.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 43805 2026-02-20T21:16:21.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:21.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:21.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:21.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:21.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:21.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:21.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:21.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:21.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:21.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:21.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:21.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:21.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:21.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:21.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=352, tag_tid=26, entry_tid=44100], [object_number=355, tag_tid=26, entry_tid=44099], [object_number=354, tag_tid=26, entry_tid=44098], [object_number=353, tag_tid=26, entry_tid=44097]]], state=connected] 2026-02-20T21:16:21.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:21.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=352, tag_tid=26, entry_tid=44100], [object_number=355, tag_tid=26, entry_tid=44099], [object_number=354, tag_tid=26, entry_tid=44098], [object_number=353, tag_tid=26, entry_tid=44097]]], state=connected] 2026-02-20T21:16:22.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=352, tag_tid=26, entry_tid=44100' 2026-02-20T21:16:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=352, tag_tid=26, entry_tid=44100' 2026-02-20T21:16:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=352, tag_tid=26, entry_tid=44100' '!=' 'object_number=353, tag_tid=26, entry_tid=43805' 2026-02-20T21:16:22.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=352, tag_tid=26, entry_tid=44100' = 'object_number=353, tag_tid=26, entry_tid=43805' 2026-02-20T21:16:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=352, tag_tid=26, entry_tid=44100' 2026-02-20T21:16:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:22.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=352, tag_tid=26, entry_tid=44100' 2026-02-20T21:16:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:22.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:22.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:22.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:22.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:22.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=352, tag_tid=26, entry_tid=44100' 2026-02-20T21:16:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:22.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44100 2026-02-20T21:16:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 44100 2026-02-20T21:16:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:22.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:22.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:22.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:22.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:22.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:22.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:22.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:22.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:22.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:22.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:22.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:22.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:22.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:22.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:22.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:22.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:22.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:22.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:22.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:22.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=352, tag_tid=26, entry_tid=44424], [object_number=355, tag_tid=26, entry_tid=44423], [object_number=354, tag_tid=26, entry_tid=44422], [object_number=353, tag_tid=26, entry_tid=44421]]], state=connected] 2026-02-20T21:16:22.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:22.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:22.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:22.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:22.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:22.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:22.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=352, tag_tid=26, entry_tid=44424], [object_number=355, tag_tid=26, entry_tid=44423], [object_number=354, tag_tid=26, entry_tid=44422], [object_number=353, tag_tid=26, entry_tid=44421]]], state=connected] 2026-02-20T21:16:22.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=352, tag_tid=26, entry_tid=44424' 2026-02-20T21:16:22.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=352, tag_tid=26, entry_tid=44424' 2026-02-20T21:16:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=352, tag_tid=26, entry_tid=44424' '!=' 'object_number=352, tag_tid=26, entry_tid=44100' 2026-02-20T21:16:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=352, tag_tid=26, entry_tid=44424' = 'object_number=352, tag_tid=26, entry_tid=44100' 2026-02-20T21:16:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=352, tag_tid=26, entry_tid=44424' 2026-02-20T21:16:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:22.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:22.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=352, tag_tid=26, entry_tid=44424' 2026-02-20T21:16:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:22.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:22.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:22.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:22.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:22.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=352, tag_tid=26, entry_tid=44424' 2026-02-20T21:16:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:22.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:22.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44424 2026-02-20T21:16:22.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 44424 2026-02-20T21:16:22.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:22.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:22.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:22.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:22.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:22.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:22.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:22.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:22.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:22.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:22.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:22.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:22.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:22.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=355, tag_tid=26, entry_tid=44707], [object_number=354, tag_tid=26, entry_tid=44706], [object_number=353, tag_tid=26, entry_tid=44705], [object_number=352, tag_tid=26, entry_tid=44704]]], state=connected] 2026-02-20T21:16:22.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:22.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:22.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:22.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:22.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:22.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:22.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:22.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:22.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:22.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:22.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:22.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=355, tag_tid=26, entry_tid=44707], [object_number=354, tag_tid=26, entry_tid=44706], [object_number=353, tag_tid=26, entry_tid=44705], [object_number=352, tag_tid=26, entry_tid=44704]]], state=connected] 2026-02-20T21:16:22.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=355, tag_tid=26, entry_tid=44707' 2026-02-20T21:16:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=355, tag_tid=26, entry_tid=44707' 2026-02-20T21:16:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=355, tag_tid=26, entry_tid=44707' '!=' 'object_number=352, tag_tid=26, entry_tid=44424' 2026-02-20T21:16:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:22.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=355, tag_tid=26, entry_tid=44707' = 'object_number=352, tag_tid=26, entry_tid=44424' 2026-02-20T21:16:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=355, tag_tid=26, entry_tid=44707' 2026-02-20T21:16:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:22.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=355, tag_tid=26, entry_tid=44707' 2026-02-20T21:16:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:22.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=355, tag_tid=26, entry_tid=44707' 2026-02-20T21:16:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:22.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=44707 2026-02-20T21:16:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 44707 2026-02-20T21:16:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:22.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:22.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:22.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:22.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:22.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:22.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:22.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:23.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:23.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:23.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:23.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=353, tag_tid=26, entry_tid=45017], [object_number=352, tag_tid=26, entry_tid=45016], [object_number=355, tag_tid=26, entry_tid=45015], [object_number=354, tag_tid=26, entry_tid=45014]]], state=connected] 2026-02-20T21:16:23.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:23.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:23.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:23.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:23.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=353, tag_tid=26, entry_tid=45017], [object_number=352, tag_tid=26, entry_tid=45016], [object_number=355, tag_tid=26, entry_tid=45015], [object_number=354, tag_tid=26, entry_tid=45014]]], state=connected] 2026-02-20T21:16:23.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=353, tag_tid=26, entry_tid=45017' 2026-02-20T21:16:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=353, tag_tid=26, entry_tid=45017' 2026-02-20T21:16:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=353, tag_tid=26, entry_tid=45017' '!=' 'object_number=355, tag_tid=26, entry_tid=44707' 2026-02-20T21:16:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=353, tag_tid=26, entry_tid=45017' = 'object_number=355, tag_tid=26, entry_tid=44707' 2026-02-20T21:16:23.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=353, tag_tid=26, entry_tid=45017' 2026-02-20T21:16:23.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:23.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:23.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:23.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:23.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=353, tag_tid=26, entry_tid=45017' 2026-02-20T21:16:23.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:23.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:23.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:23.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:23.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:23.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:23.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:23.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=353, tag_tid=26, entry_tid=45017' 2026-02-20T21:16:23.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:23.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:23.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45017 2026-02-20T21:16:23.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 45017 2026-02-20T21:16:23.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:23.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:23.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:23.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:23.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:23.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:23.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:23.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:23.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:23.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:23.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:23.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:23.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:23.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:23.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:23.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:23.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:23.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:23.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:23.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=352, tag_tid=26, entry_tid=45296], [object_number=355, tag_tid=26, entry_tid=45295], [object_number=354, tag_tid=26, entry_tid=45294], [object_number=353, tag_tid=26, entry_tid=45293]]], state=connected] 2026-02-20T21:16:23.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:23.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:23.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:23.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=352, tag_tid=26, entry_tid=45296], [object_number=355, tag_tid=26, entry_tid=45295], [object_number=354, tag_tid=26, entry_tid=45294], [object_number=353, tag_tid=26, entry_tid=45293]]], state=connected] 2026-02-20T21:16:23.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=352, tag_tid=26, entry_tid=45296' 2026-02-20T21:16:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=352, tag_tid=26, entry_tid=45296' 2026-02-20T21:16:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=352, tag_tid=26, entry_tid=45296' '!=' 'object_number=353, tag_tid=26, entry_tid=45017' 2026-02-20T21:16:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=352, tag_tid=26, entry_tid=45296' = 'object_number=353, tag_tid=26, entry_tid=45017' 2026-02-20T21:16:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=352, tag_tid=26, entry_tid=45296' 2026-02-20T21:16:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:23.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:23.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=352, tag_tid=26, entry_tid=45296' 2026-02-20T21:16:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:23.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=352, tag_tid=26, entry_tid=45296' 2026-02-20T21:16:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45296 2026-02-20T21:16:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 45296 2026-02-20T21:16:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:23.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:23.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:23.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:23.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:23.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:23.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:23.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:23.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=354, tag_tid=26, entry_tid=45594], [object_number=353, tag_tid=26, entry_tid=45593], [object_number=352, tag_tid=26, entry_tid=45592], [object_number=355, tag_tid=26, entry_tid=45591]]], state=connected] 2026-02-20T21:16:23.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:23.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:23.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:23.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:23.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:23.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=354, tag_tid=26, entry_tid=45594], [object_number=353, tag_tid=26, entry_tid=45593], [object_number=352, tag_tid=26, entry_tid=45592], [object_number=355, tag_tid=26, entry_tid=45591]]], state=connected] 2026-02-20T21:16:23.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=354, tag_tid=26, entry_tid=45594' 2026-02-20T21:16:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=354, tag_tid=26, entry_tid=45594' 2026-02-20T21:16:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=354, tag_tid=26, entry_tid=45594' '!=' 'object_number=352, tag_tid=26, entry_tid=45296' 2026-02-20T21:16:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=354, tag_tid=26, entry_tid=45594' = 'object_number=352, tag_tid=26, entry_tid=45296' 2026-02-20T21:16:23.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=354, tag_tid=26, entry_tid=45594' 2026-02-20T21:16:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:23.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:23.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=354, tag_tid=26, entry_tid=45594' 2026-02-20T21:16:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:23.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:23.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:23.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:23.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:23.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=354, tag_tid=26, entry_tid=45594' 2026-02-20T21:16:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45594 2026-02-20T21:16:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 45594 2026-02-20T21:16:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:23.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:23.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:23.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:23.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:23.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:23.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:23.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:23.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:23.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:23.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:23.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:23.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:23.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:23.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:23.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:23.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:23.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:23.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:23.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:23.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:23.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:23.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:23.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:23.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:23.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:23.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:23.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:24.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:24.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:24.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:24.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=352, tag_tid=26, entry_tid=45896], [object_number=355, tag_tid=26, entry_tid=45895], [object_number=354, tag_tid=26, entry_tid=45894], [object_number=353, tag_tid=26, entry_tid=45893]]], state=connected] 2026-02-20T21:16:24.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:24.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=352, tag_tid=26, entry_tid=45896], [object_number=355, tag_tid=26, entry_tid=45895], [object_number=354, tag_tid=26, entry_tid=45894], [object_number=353, tag_tid=26, entry_tid=45893]]], state=connected] 2026-02-20T21:16:24.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=352, tag_tid=26, entry_tid=45896' 2026-02-20T21:16:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=352, tag_tid=26, entry_tid=45896' 2026-02-20T21:16:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=352, tag_tid=26, entry_tid=45896' '!=' 'object_number=354, tag_tid=26, entry_tid=45594' 2026-02-20T21:16:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=352, tag_tid=26, entry_tid=45896' = 'object_number=354, tag_tid=26, entry_tid=45594' 2026-02-20T21:16:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=352, tag_tid=26, entry_tid=45896' 2026-02-20T21:16:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=352, tag_tid=26, entry_tid=45896' 2026-02-20T21:16:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=352, tag_tid=26, entry_tid=45896' 2026-02-20T21:16:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45896 2026-02-20T21:16:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 45896 2026-02-20T21:16:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:24.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:24.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:24.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:24.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:24.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:24.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:24.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:24.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:24.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:24.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:24.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:24.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:24.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:24.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:24.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:24.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:24.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:24.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:24.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:24.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:24.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:24.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:24.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:24.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:24.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:24.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:24.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:24.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:24.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:24.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:24.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=355, tag_tid=26, entry_tid=46187], [object_number=354, tag_tid=26, entry_tid=46186], [object_number=353, tag_tid=26, entry_tid=46185], [object_number=352, tag_tid=26, entry_tid=46184]]], state=connected] 2026-02-20T21:16:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:24.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:24.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:24.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:24.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:24.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:24.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:24.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=355, tag_tid=26, entry_tid=46187], [object_number=354, tag_tid=26, entry_tid=46186], [object_number=353, tag_tid=26, entry_tid=46185], [object_number=352, tag_tid=26, entry_tid=46184]]], state=connected] 2026-02-20T21:16:24.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=355, tag_tid=26, entry_tid=46187' 2026-02-20T21:16:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=355, tag_tid=26, entry_tid=46187' 2026-02-20T21:16:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=355, tag_tid=26, entry_tid=46187' '!=' 'object_number=352, tag_tid=26, entry_tid=45896' 2026-02-20T21:16:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=355, tag_tid=26, entry_tid=46187' = 'object_number=352, tag_tid=26, entry_tid=45896' 2026-02-20T21:16:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=355, tag_tid=26, entry_tid=46187' 2026-02-20T21:16:24.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:24.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:24.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:24.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:24.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=355, tag_tid=26, entry_tid=46187' 2026-02-20T21:16:24.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:24.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:24.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:24.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:24.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:24.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=355, tag_tid=26, entry_tid=46187' 2026-02-20T21:16:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:24.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46187 2026-02-20T21:16:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 46187 2026-02-20T21:16:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:24.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:24.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:24.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:24.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:24.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:24.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:24.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:24.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:24.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:24.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:24.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:24.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:24.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:24.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=353, tag_tid=26, entry_tid=46489], [object_number=352, tag_tid=26, entry_tid=46488], [object_number=355, tag_tid=26, entry_tid=46487], [object_number=354, tag_tid=26, entry_tid=46486]]], state=connected] 2026-02-20T21:16:24.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:24.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:24.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:24.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:24.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:24.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=353, tag_tid=26, entry_tid=46489], [object_number=352, tag_tid=26, entry_tid=46488], [object_number=355, tag_tid=26, entry_tid=46487], [object_number=354, tag_tid=26, entry_tid=46486]]], state=connected] 2026-02-20T21:16:24.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=353, tag_tid=26, entry_tid=46489' 2026-02-20T21:16:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=353, tag_tid=26, entry_tid=46489' 2026-02-20T21:16:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=353, tag_tid=26, entry_tid=46489' '!=' 'object_number=355, tag_tid=26, entry_tid=46187' 2026-02-20T21:16:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=353, tag_tid=26, entry_tid=46489' = 'object_number=355, tag_tid=26, entry_tid=46187' 2026-02-20T21:16:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=353, tag_tid=26, entry_tid=46489' 2026-02-20T21:16:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:24.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=353, tag_tid=26, entry_tid=46489' 2026-02-20T21:16:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:24.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:24.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:24.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:24.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=353, tag_tid=26, entry_tid=46489' 2026-02-20T21:16:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:24.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46489 2026-02-20T21:16:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 46489 2026-02-20T21:16:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:24.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:24.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:24.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:24.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:24.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:24.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:24.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:24.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:24.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:24.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:24.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:24.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:24.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:24.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:25.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:25.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:25.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:25.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=352, tag_tid=26, entry_tid=46800], [object_number=355, tag_tid=26, entry_tid=46799], [object_number=354, tag_tid=26, entry_tid=46798], [object_number=353, tag_tid=26, entry_tid=46797]]], state=connected] 2026-02-20T21:16:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=352, tag_tid=26, entry_tid=46800], [object_number=355, tag_tid=26, entry_tid=46799], [object_number=354, tag_tid=26, entry_tid=46798], [object_number=353, tag_tid=26, entry_tid=46797]]], state=connected] 2026-02-20T21:16:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=352, tag_tid=26, entry_tid=46800' 2026-02-20T21:16:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=352, tag_tid=26, entry_tid=46800' 2026-02-20T21:16:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=352, tag_tid=26, entry_tid=46800' '!=' 'object_number=353, tag_tid=26, entry_tid=46489' 2026-02-20T21:16:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=352, tag_tid=26, entry_tid=46800' = 'object_number=353, tag_tid=26, entry_tid=46489' 2026-02-20T21:16:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=352, tag_tid=26, entry_tid=46800' 2026-02-20T21:16:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:25.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:25.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:25.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=352, tag_tid=26, entry_tid=46800' 2026-02-20T21:16:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:25.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=352, tag_tid=26, entry_tid=46800' 2026-02-20T21:16:25.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:25.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:25.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46800 2026-02-20T21:16:25.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 46800 2026-02-20T21:16:25.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:25.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:25.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:25.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:25.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:25.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:25.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:25.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:25.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:25.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:25.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:25.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:25.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:25.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=353, tag_tid=26, entry_tid=47097], [object_number=352, tag_tid=26, entry_tid=47096], [object_number=355, tag_tid=26, entry_tid=47095], [object_number=354, tag_tid=26, entry_tid=47094]]], state=connected] 2026-02-20T21:16:25.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=353, tag_tid=26, entry_tid=47097], [object_number=352, tag_tid=26, entry_tid=47096], [object_number=355, tag_tid=26, entry_tid=47095], [object_number=354, tag_tid=26, entry_tid=47094]]], state=connected] 2026-02-20T21:16:25.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=353, tag_tid=26, entry_tid=47097' 2026-02-20T21:16:25.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=353, tag_tid=26, entry_tid=47097' 2026-02-20T21:16:25.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=353, tag_tid=26, entry_tid=47097' '!=' 'object_number=352, tag_tid=26, entry_tid=46800' 2026-02-20T21:16:25.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:25.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=353, tag_tid=26, entry_tid=47097' = 'object_number=352, tag_tid=26, entry_tid=46800' 2026-02-20T21:16:25.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=353, tag_tid=26, entry_tid=47097' 2026-02-20T21:16:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:25.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:25.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=353, tag_tid=26, entry_tid=47097' 2026-02-20T21:16:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:25.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=353, tag_tid=26, entry_tid=47097' 2026-02-20T21:16:25.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:25.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47097 2026-02-20T21:16:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 47097 2026-02-20T21:16:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:25.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:25.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:25.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:25.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:25.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:25.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:25.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:25.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:25.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:25.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:25.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:25.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:25.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:25.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:25.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:25.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:25.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:25.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:25.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:25.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=355, tag_tid=26, entry_tid=47399], [object_number=354, tag_tid=26, entry_tid=47398], [object_number=353, tag_tid=26, entry_tid=47397], [object_number=352, tag_tid=26, entry_tid=47396]]], state=connected] 2026-02-20T21:16:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:25.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:25.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 88 2026-02-20T21:16:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=355, tag_tid=26, entry_tid=47399], [object_number=354, tag_tid=26, entry_tid=47398], [object_number=353, tag_tid=26, entry_tid=47397], [object_number=352, tag_tid=26, entry_tid=47396]]], state=connected] 2026-02-20T21:16:25.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=355, tag_tid=26, entry_tid=47399' 2026-02-20T21:16:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=355, tag_tid=26, entry_tid=47399' 2026-02-20T21:16:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=355, tag_tid=26, entry_tid=47399' '!=' 'object_number=353, tag_tid=26, entry_tid=47097' 2026-02-20T21:16:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:25.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=355, tag_tid=26, entry_tid=47399' = 'object_number=353, tag_tid=26, entry_tid=47097' 2026-02-20T21:16:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=355, tag_tid=26, entry_tid=47399' 2026-02-20T21:16:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:25.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:25.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:25.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=355, tag_tid=26, entry_tid=47399' 2026-02-20T21:16:25.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:25.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:25.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:25.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:25.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=355, tag_tid=26, entry_tid=47399' 2026-02-20T21:16:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:25.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47399 2026-02-20T21:16:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 47399 2026-02-20T21:16:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:25.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:26.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:26.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:26.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:26.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:26.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:26.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:26.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:26.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:26.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:26.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:26.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:26.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:26.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:26.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:26.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:26.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:26.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=356, tag_tid=26, entry_tid=47612], [object_number=359, tag_tid=26, entry_tid=47611], [object_number=358, tag_tid=26, entry_tid=47610], [object_number=357, tag_tid=26, entry_tid=47609]]], state=connected] 2026-02-20T21:16:26.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:26.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=356, tag_tid=26, entry_tid=47612], [object_number=359, tag_tid=26, entry_tid=47611], [object_number=358, tag_tid=26, entry_tid=47610], [object_number=357, tag_tid=26, entry_tid=47609]]], state=connected] 2026-02-20T21:16:26.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=356, tag_tid=26, entry_tid=47612' 2026-02-20T21:16:26.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=356, tag_tid=26, entry_tid=47612' 2026-02-20T21:16:26.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=356, tag_tid=26, entry_tid=47612' '!=' 'object_number=355, tag_tid=26, entry_tid=47399' 2026-02-20T21:16:26.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:26.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=356, tag_tid=26, entry_tid=47612' = 'object_number=355, tag_tid=26, entry_tid=47399' 2026-02-20T21:16:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=356, tag_tid=26, entry_tid=47612' 2026-02-20T21:16:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:26.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:26.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=356, tag_tid=26, entry_tid=47612' 2026-02-20T21:16:26.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:26.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:26.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:26.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:26.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:26.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:26.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:26.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=356, tag_tid=26, entry_tid=47612' 2026-02-20T21:16:26.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:26.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47612 2026-02-20T21:16:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 47612 2026-02-20T21:16:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:26.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:26.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:26.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:26.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:26.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:26.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:26.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:26.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:26.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:26.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:26.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:26.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:26.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:26.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:26.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:26.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:26.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:26.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:26.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:26.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=358, tag_tid=26, entry_tid=47878], [object_number=357, tag_tid=26, entry_tid=47877], [object_number=356, tag_tid=26, entry_tid=47876], [object_number=359, tag_tid=26, entry_tid=47875]]], state=connected] 2026-02-20T21:16:26.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:26.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:26.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:26.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:26.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:26.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:26.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:26.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:26.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:26.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=358, tag_tid=26, entry_tid=47878], [object_number=357, tag_tid=26, entry_tid=47877], [object_number=356, tag_tid=26, entry_tid=47876], [object_number=359, tag_tid=26, entry_tid=47875]]], state=connected] 2026-02-20T21:16:26.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=358, tag_tid=26, entry_tid=47878' 2026-02-20T21:16:26.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=358, tag_tid=26, entry_tid=47878' 2026-02-20T21:16:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=358, tag_tid=26, entry_tid=47878' '!=' 'object_number=356, tag_tid=26, entry_tid=47612' 2026-02-20T21:16:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=358, tag_tid=26, entry_tid=47878' = 'object_number=356, tag_tid=26, entry_tid=47612' 2026-02-20T21:16:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=358, tag_tid=26, entry_tid=47878' 2026-02-20T21:16:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:26.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:26.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:26.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=358, tag_tid=26, entry_tid=47878' 2026-02-20T21:16:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:26.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:26.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:26.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:26.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:26.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=358, tag_tid=26, entry_tid=47878' 2026-02-20T21:16:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:26.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47878 2026-02-20T21:16:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 47878 2026-02-20T21:16:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:26.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:26.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:26.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:26.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:26.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:26.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:26.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:26.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:26.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:26.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=359, tag_tid=26, entry_tid=48359], [object_number=358, tag_tid=26, entry_tid=48358], [object_number=357, tag_tid=26, entry_tid=48357], [object_number=356, tag_tid=26, entry_tid=48356]]], state=connected] 2026-02-20T21:16:26.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:26.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:27.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=359, tag_tid=26, entry_tid=48359], [object_number=358, tag_tid=26, entry_tid=48358], [object_number=357, tag_tid=26, entry_tid=48357], [object_number=356, tag_tid=26, entry_tid=48356]]], state=connected] 2026-02-20T21:16:27.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=359, tag_tid=26, entry_tid=48359' 2026-02-20T21:16:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=359, tag_tid=26, entry_tid=48359' 2026-02-20T21:16:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=359, tag_tid=26, entry_tid=48359' '!=' 'object_number=358, tag_tid=26, entry_tid=47878' 2026-02-20T21:16:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:27.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=359, tag_tid=26, entry_tid=48359' = 'object_number=358, tag_tid=26, entry_tid=47878' 2026-02-20T21:16:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=359, tag_tid=26, entry_tid=48359' 2026-02-20T21:16:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:27.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=359, tag_tid=26, entry_tid=48359' 2026-02-20T21:16:27.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:27.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:27.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:27.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:27.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:27.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:27.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:27.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=359, tag_tid=26, entry_tid=48359' 2026-02-20T21:16:27.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:27.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:27.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48359 2026-02-20T21:16:27.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 48359 2026-02-20T21:16:27.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:27.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:27.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:27.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:27.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:27.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:27.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:27.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:27.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=359, tag_tid=26, entry_tid=48659], [object_number=358, tag_tid=26, entry_tid=48658], [object_number=357, tag_tid=26, entry_tid=48657], [object_number=356, tag_tid=26, entry_tid=48656]]], state=connected] 2026-02-20T21:16:27.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:27.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:27.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:27.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:27.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:27.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:27.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=359, tag_tid=26, entry_tid=48659], [object_number=358, tag_tid=26, entry_tid=48658], [object_number=357, tag_tid=26, entry_tid=48657], [object_number=356, tag_tid=26, entry_tid=48656]]], state=connected] 2026-02-20T21:16:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=359, tag_tid=26, entry_tid=48659' 2026-02-20T21:16:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=359, tag_tid=26, entry_tid=48659' 2026-02-20T21:16:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=359, tag_tid=26, entry_tid=48659' '!=' 'object_number=359, tag_tid=26, entry_tid=48359' 2026-02-20T21:16:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=359, tag_tid=26, entry_tid=48659' = 'object_number=359, tag_tid=26, entry_tid=48359' 2026-02-20T21:16:27.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=359, tag_tid=26, entry_tid=48659' 2026-02-20T21:16:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:27.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:27.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=359, tag_tid=26, entry_tid=48659' 2026-02-20T21:16:27.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:27.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:27.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:27.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:27.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:27.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:27.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:27.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=359, tag_tid=26, entry_tid=48659' 2026-02-20T21:16:27.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:27.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48659 2026-02-20T21:16:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 48659 2026-02-20T21:16:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:27.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:27.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:27.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:27.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:27.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:27.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:27.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:27.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:27.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:27.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:27.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:27.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:27.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:27.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:27.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:27.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:27.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:27.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:27.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:27.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:27.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:27.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:27.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:27.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:27.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:27.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:27.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:27.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=356, tag_tid=26, entry_tid=48940], [object_number=359, tag_tid=26, entry_tid=48939], [object_number=358, tag_tid=26, entry_tid=48938], [object_number=357, tag_tid=26, entry_tid=48937]]], state=connected] 2026-02-20T21:16:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:27.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:27.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:27.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:27.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:27.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:27.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:27.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:27.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=356, tag_tid=26, entry_tid=48940], [object_number=359, tag_tid=26, entry_tid=48939], [object_number=358, tag_tid=26, entry_tid=48938], [object_number=357, tag_tid=26, entry_tid=48937]]], state=connected] 2026-02-20T21:16:27.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=356, tag_tid=26, entry_tid=48940' 2026-02-20T21:16:27.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=356, tag_tid=26, entry_tid=48940' 2026-02-20T21:16:27.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=356, tag_tid=26, entry_tid=48940' '!=' 'object_number=359, tag_tid=26, entry_tid=48659' 2026-02-20T21:16:27.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:27.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=356, tag_tid=26, entry_tid=48940' = 'object_number=359, tag_tid=26, entry_tid=48659' 2026-02-20T21:16:27.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=356, tag_tid=26, entry_tid=48940' 2026-02-20T21:16:27.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:27.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:27.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:27.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:27.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=356, tag_tid=26, entry_tid=48940' 2026-02-20T21:16:27.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:27.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:27.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:27.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:27.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:27.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:27.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=356, tag_tid=26, entry_tid=48940' 2026-02-20T21:16:27.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:27.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:27.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=48940 2026-02-20T21:16:27.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 48940 2026-02-20T21:16:27.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:27.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:27.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:27.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:27.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:27.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:27.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:27.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:27.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:27.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:27.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:27.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:27.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:27.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:27.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:27.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:27.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:27.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:27.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=358, tag_tid=26, entry_tid=49174], [object_number=357, tag_tid=26, entry_tid=49173], [object_number=356, tag_tid=26, entry_tid=49172], [object_number=359, tag_tid=26, entry_tid=49171]]], state=connected] 2026-02-20T21:16:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:28.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:28.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:28.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=358, tag_tid=26, entry_tid=49174], [object_number=357, tag_tid=26, entry_tid=49173], [object_number=356, tag_tid=26, entry_tid=49172], [object_number=359, tag_tid=26, entry_tid=49171]]], state=connected] 2026-02-20T21:16:28.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=358, tag_tid=26, entry_tid=49174' 2026-02-20T21:16:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=358, tag_tid=26, entry_tid=49174' 2026-02-20T21:16:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=358, tag_tid=26, entry_tid=49174' '!=' 'object_number=356, tag_tid=26, entry_tid=48940' 2026-02-20T21:16:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=358, tag_tid=26, entry_tid=49174' = 'object_number=356, tag_tid=26, entry_tid=48940' 2026-02-20T21:16:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=358, tag_tid=26, entry_tid=49174' 2026-02-20T21:16:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:28.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=358, tag_tid=26, entry_tid=49174' 2026-02-20T21:16:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:28.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:28.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:28.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:28.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:28.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:28.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=358, tag_tid=26, entry_tid=49174' 2026-02-20T21:16:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49174 2026-02-20T21:16:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 49174 2026-02-20T21:16:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:28.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:28.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:28.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:28.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:28.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:28.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:28.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:28.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:28.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:28.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:28.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:28.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:28.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:28.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:28.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:28.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:28.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:28.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:28.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:28.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=357, tag_tid=26, entry_tid=49445], [object_number=356, tag_tid=26, entry_tid=49444], [object_number=359, tag_tid=26, entry_tid=49443], [object_number=358, tag_tid=26, entry_tid=49442]]], state=connected] 2026-02-20T21:16:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:28.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=357, tag_tid=26, entry_tid=49445], [object_number=356, tag_tid=26, entry_tid=49444], [object_number=359, tag_tid=26, entry_tid=49443], [object_number=358, tag_tid=26, entry_tid=49442]]], state=connected] 2026-02-20T21:16:28.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=357, tag_tid=26, entry_tid=49445' 2026-02-20T21:16:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=357, tag_tid=26, entry_tid=49445' 2026-02-20T21:16:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=357, tag_tid=26, entry_tid=49445' '!=' 'object_number=358, tag_tid=26, entry_tid=49174' 2026-02-20T21:16:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=357, tag_tid=26, entry_tid=49445' = 'object_number=358, tag_tid=26, entry_tid=49174' 2026-02-20T21:16:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=357, tag_tid=26, entry_tid=49445' 2026-02-20T21:16:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=357, tag_tid=26, entry_tid=49445' 2026-02-20T21:16:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:28.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:28.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:28.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:28.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:28.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=357, tag_tid=26, entry_tid=49445' 2026-02-20T21:16:28.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:28.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:28.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49445 2026-02-20T21:16:28.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 49445 2026-02-20T21:16:28.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:28.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:28.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:28.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:28.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:28.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:28.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:28.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:28.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:28.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:28.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:28.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:28.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:28.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:28.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:28.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:28.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:28.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:28.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:28.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:28.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:28.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:28.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:28.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:28.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:28.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=358, tag_tid=26, entry_tid=49722], [object_number=357, tag_tid=26, entry_tid=49721], [object_number=356, tag_tid=26, entry_tid=49720], [object_number=359, tag_tid=26, entry_tid=49719]]], state=connected] 2026-02-20T21:16:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:28.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:28.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:28.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:28.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:28.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:28.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:28.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:28.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:28.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:28.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:28.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=358, tag_tid=26, entry_tid=49722], [object_number=357, tag_tid=26, entry_tid=49721], [object_number=356, tag_tid=26, entry_tid=49720], [object_number=359, tag_tid=26, entry_tid=49719]]], state=connected] 2026-02-20T21:16:28.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:28.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=358, tag_tid=26, entry_tid=49722' 2026-02-20T21:16:28.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=358, tag_tid=26, entry_tid=49722' 2026-02-20T21:16:28.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=358, tag_tid=26, entry_tid=49722' '!=' 'object_number=357, tag_tid=26, entry_tid=49445' 2026-02-20T21:16:28.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:28.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=358, tag_tid=26, entry_tid=49722' = 'object_number=357, tag_tid=26, entry_tid=49445' 2026-02-20T21:16:28.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=358, tag_tid=26, entry_tid=49722' 2026-02-20T21:16:28.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:28.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:28.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:28.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:28.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=358, tag_tid=26, entry_tid=49722' 2026-02-20T21:16:28.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:28.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=358, tag_tid=26, entry_tid=49722' 2026-02-20T21:16:28.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:28.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:28.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49722 2026-02-20T21:16:28.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 49722 2026-02-20T21:16:28.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:28.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:28.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:28.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:28.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:28.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:29.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:29.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:29.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:29.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=357, tag_tid=26, entry_tid=50017], [object_number=356, tag_tid=26, entry_tid=50016], [object_number=359, tag_tid=26, entry_tid=50015], [object_number=358, tag_tid=26, entry_tid=50014]]], state=connected] 2026-02-20T21:16:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:29.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:29.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:29.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:29.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=357, tag_tid=26, entry_tid=50017], [object_number=356, tag_tid=26, entry_tid=50016], [object_number=359, tag_tid=26, entry_tid=50015], [object_number=358, tag_tid=26, entry_tid=50014]]], state=connected] 2026-02-20T21:16:29.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=357, tag_tid=26, entry_tid=50017' 2026-02-20T21:16:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=357, tag_tid=26, entry_tid=50017' 2026-02-20T21:16:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=357, tag_tid=26, entry_tid=50017' '!=' 'object_number=358, tag_tid=26, entry_tid=49722' 2026-02-20T21:16:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=357, tag_tid=26, entry_tid=50017' = 'object_number=358, tag_tid=26, entry_tid=49722' 2026-02-20T21:16:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=357, tag_tid=26, entry_tid=50017' 2026-02-20T21:16:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:29.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:29.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:29.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=357, tag_tid=26, entry_tid=50017' 2026-02-20T21:16:29.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:29.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:29.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:29.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:29.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:29.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=357, tag_tid=26, entry_tid=50017' 2026-02-20T21:16:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50017 2026-02-20T21:16:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 50017 2026-02-20T21:16:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:29.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:29.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:29.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:29.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:29.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:29.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:29.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:29.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:29.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:29.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:29.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:29.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:29.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:29.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:29.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:29.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:29.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:29.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=357, tag_tid=26, entry_tid=50317], [object_number=356, tag_tid=26, entry_tid=50316], [object_number=359, tag_tid=26, entry_tid=50315], [object_number=358, tag_tid=26, entry_tid=50314]]], state=connected] 2026-02-20T21:16:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:29.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:29.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:29.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:29.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=357, tag_tid=26, entry_tid=50317], [object_number=356, tag_tid=26, entry_tid=50316], [object_number=359, tag_tid=26, entry_tid=50315], [object_number=358, tag_tid=26, entry_tid=50314]]], state=connected] 2026-02-20T21:16:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=357, tag_tid=26, entry_tid=50317' 2026-02-20T21:16:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=357, tag_tid=26, entry_tid=50317' 2026-02-20T21:16:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=357, tag_tid=26, entry_tid=50317' '!=' 'object_number=357, tag_tid=26, entry_tid=50017' 2026-02-20T21:16:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=357, tag_tid=26, entry_tid=50317' = 'object_number=357, tag_tid=26, entry_tid=50017' 2026-02-20T21:16:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=357, tag_tid=26, entry_tid=50317' 2026-02-20T21:16:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=357, tag_tid=26, entry_tid=50317' 2026-02-20T21:16:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:29.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:29.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=357, tag_tid=26, entry_tid=50317' 2026-02-20T21:16:29.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:29.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50317 2026-02-20T21:16:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 50317 2026-02-20T21:16:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:29.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:29.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:29.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:29.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:29.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:29.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:29.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:29.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:29.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:29.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:29.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=356, tag_tid=26, entry_tid=50620], [object_number=359, tag_tid=26, entry_tid=50619], [object_number=358, tag_tid=26, entry_tid=50618], [object_number=357, tag_tid=26, entry_tid=50617]]], state=connected] 2026-02-20T21:16:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:29.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:29.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:29.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:29.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:29.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:29.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:29.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=356, tag_tid=26, entry_tid=50620], [object_number=359, tag_tid=26, entry_tid=50619], [object_number=358, tag_tid=26, entry_tid=50618], [object_number=357, tag_tid=26, entry_tid=50617]]], state=connected] 2026-02-20T21:16:29.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=356, tag_tid=26, entry_tid=50620' 2026-02-20T21:16:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=356, tag_tid=26, entry_tid=50620' 2026-02-20T21:16:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=356, tag_tid=26, entry_tid=50620' '!=' 'object_number=357, tag_tid=26, entry_tid=50317' 2026-02-20T21:16:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=356, tag_tid=26, entry_tid=50620' = 'object_number=357, tag_tid=26, entry_tid=50317' 2026-02-20T21:16:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=356, tag_tid=26, entry_tid=50620' 2026-02-20T21:16:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:29.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:29.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:29.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=356, tag_tid=26, entry_tid=50620' 2026-02-20T21:16:29.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:29.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:29.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:29.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:29.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:29.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:29.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=356, tag_tid=26, entry_tid=50620' 2026-02-20T21:16:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50620 2026-02-20T21:16:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 50620 2026-02-20T21:16:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:29.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:29.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:29.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:29.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:29.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:29.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:30.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:30.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:30.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=359, tag_tid=26, entry_tid=50915], [object_number=358, tag_tid=26, entry_tid=50914], [object_number=357, tag_tid=26, entry_tid=50913], [object_number=356, tag_tid=26, entry_tid=50912]]], state=connected] 2026-02-20T21:16:30.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:30.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:30.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:30.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:30.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=359, tag_tid=26, entry_tid=50915], [object_number=358, tag_tid=26, entry_tid=50914], [object_number=357, tag_tid=26, entry_tid=50913], [object_number=356, tag_tid=26, entry_tid=50912]]], state=connected] 2026-02-20T21:16:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=359, tag_tid=26, entry_tid=50915' 2026-02-20T21:16:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=359, tag_tid=26, entry_tid=50915' 2026-02-20T21:16:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=359, tag_tid=26, entry_tid=50915' '!=' 'object_number=356, tag_tid=26, entry_tid=50620' 2026-02-20T21:16:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=359, tag_tid=26, entry_tid=50915' = 'object_number=356, tag_tid=26, entry_tid=50620' 2026-02-20T21:16:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=359, tag_tid=26, entry_tid=50915' 2026-02-20T21:16:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=359, tag_tid=26, entry_tid=50915' 2026-02-20T21:16:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:30.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:30.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=359, tag_tid=26, entry_tid=50915' 2026-02-20T21:16:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=50915 2026-02-20T21:16:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 50915 2026-02-20T21:16:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:30.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:30.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:30.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:30.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:30.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:30.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:30.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:30.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:30.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=359, tag_tid=26, entry_tid=51203], [object_number=358, tag_tid=26, entry_tid=51202], [object_number=357, tag_tid=26, entry_tid=51201], [object_number=356, tag_tid=26, entry_tid=51200]]], state=connected] 2026-02-20T21:16:30.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:30.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=359, tag_tid=26, entry_tid=51203], [object_number=358, tag_tid=26, entry_tid=51202], [object_number=357, tag_tid=26, entry_tid=51201], [object_number=356, tag_tid=26, entry_tid=51200]]], state=connected] 2026-02-20T21:16:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=359, tag_tid=26, entry_tid=51203' 2026-02-20T21:16:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=359, tag_tid=26, entry_tid=51203' 2026-02-20T21:16:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=359, tag_tid=26, entry_tid=51203' '!=' 'object_number=359, tag_tid=26, entry_tid=50915' 2026-02-20T21:16:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=359, tag_tid=26, entry_tid=51203' = 'object_number=359, tag_tid=26, entry_tid=50915' 2026-02-20T21:16:30.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=359, tag_tid=26, entry_tid=51203' 2026-02-20T21:16:30.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:30.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:30.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:30.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=359, tag_tid=26, entry_tid=51203' 2026-02-20T21:16:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=359, tag_tid=26, entry_tid=51203' 2026-02-20T21:16:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:30.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51203 2026-02-20T21:16:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 51203 2026-02-20T21:16:30.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:30.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:30.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:30.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:30.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:30.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:30.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:30.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:30.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:30.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:30.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:30.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:30.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:30.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:30.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:30.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:30.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:30.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:30.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:30.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=358, tag_tid=26, entry_tid=51502], [object_number=357, tag_tid=26, entry_tid=51501], [object_number=356, tag_tid=26, entry_tid=51500], [object_number=359, tag_tid=26, entry_tid=51499]]], state=connected] 2026-02-20T21:16:30.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:30.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:30.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:30.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:30.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:30.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:30.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:30.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:30.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:30.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:30.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:30.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:30.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:30.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:30.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 89 2026-02-20T21:16:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:30.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=358, tag_tid=26, entry_tid=51502], [object_number=357, tag_tid=26, entry_tid=51501], [object_number=356, tag_tid=26, entry_tid=51500], [object_number=359, tag_tid=26, entry_tid=51499]]], state=connected] 2026-02-20T21:16:30.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:30.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=358, tag_tid=26, entry_tid=51502' 2026-02-20T21:16:30.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=358, tag_tid=26, entry_tid=51502' 2026-02-20T21:16:30.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=358, tag_tid=26, entry_tid=51502' '!=' 'object_number=359, tag_tid=26, entry_tid=51203' 2026-02-20T21:16:30.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:30.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=358, tag_tid=26, entry_tid=51502' = 'object_number=359, tag_tid=26, entry_tid=51203' 2026-02-20T21:16:30.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=358, tag_tid=26, entry_tid=51502' 2026-02-20T21:16:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:30.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:30.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=358, tag_tid=26, entry_tid=51502' 2026-02-20T21:16:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:30.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:30.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:30.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:30.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:30.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:30.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:30.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=358, tag_tid=26, entry_tid=51502' 2026-02-20T21:16:30.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:30.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:30.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51502 2026-02-20T21:16:30.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 51502 2026-02-20T21:16:30.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:30.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:30.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:31.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:31.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:31.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:31.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:31.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:31.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:31.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:31.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:31.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:31.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:31.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=51909], [object_number=360, tag_tid=26, entry_tid=51908], [object_number=363, tag_tid=26, entry_tid=51907], [object_number=362, tag_tid=26, entry_tid=51906]]], state=connected] 2026-02-20T21:16:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:31.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:31.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:31.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=51909], [object_number=360, tag_tid=26, entry_tid=51908], [object_number=363, tag_tid=26, entry_tid=51907], [object_number=362, tag_tid=26, entry_tid=51906]]], state=connected] 2026-02-20T21:16:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=51909' 2026-02-20T21:16:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=51909' 2026-02-20T21:16:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=51909' '!=' 'object_number=358, tag_tid=26, entry_tid=51502' 2026-02-20T21:16:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:31.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=51909' = 'object_number=358, tag_tid=26, entry_tid=51502' 2026-02-20T21:16:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=51909' 2026-02-20T21:16:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=51909' 2026-02-20T21:16:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:31.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=51909' 2026-02-20T21:16:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:31.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=51909 2026-02-20T21:16:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 51909 2026-02-20T21:16:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:31.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:31.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:31.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:31.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:31.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:31.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:31.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:31.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:31.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:31.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:31.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=363, tag_tid=26, entry_tid=52311], [object_number=362, tag_tid=26, entry_tid=52310], [object_number=361, tag_tid=26, entry_tid=52309], [object_number=360, tag_tid=26, entry_tid=52308]]], state=connected] 2026-02-20T21:16:31.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:31.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:31.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:31.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:31.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=363, tag_tid=26, entry_tid=52311], [object_number=362, tag_tid=26, entry_tid=52310], [object_number=361, tag_tid=26, entry_tid=52309], [object_number=360, tag_tid=26, entry_tid=52308]]], state=connected] 2026-02-20T21:16:31.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=363, tag_tid=26, entry_tid=52311' 2026-02-20T21:16:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=363, tag_tid=26, entry_tid=52311' 2026-02-20T21:16:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=363, tag_tid=26, entry_tid=52311' '!=' 'object_number=361, tag_tid=26, entry_tid=51909' 2026-02-20T21:16:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=363, tag_tid=26, entry_tid=52311' = 'object_number=361, tag_tid=26, entry_tid=51909' 2026-02-20T21:16:31.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=363, tag_tid=26, entry_tid=52311' 2026-02-20T21:16:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:31.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=363, tag_tid=26, entry_tid=52311' 2026-02-20T21:16:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:31.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:31.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:31.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:31.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=363, tag_tid=26, entry_tid=52311' 2026-02-20T21:16:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:31.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52311 2026-02-20T21:16:31.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 52311 2026-02-20T21:16:31.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:31.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:31.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:31.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:31.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:31.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:31.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:31.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:31.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:31.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=52669], [object_number=360, tag_tid=26, entry_tid=52668], [object_number=363, tag_tid=26, entry_tid=52667], [object_number=362, tag_tid=26, entry_tid=52666]]], state=connected] 2026-02-20T21:16:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:31.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:32.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=52669], [object_number=360, tag_tid=26, entry_tid=52668], [object_number=363, tag_tid=26, entry_tid=52667], [object_number=362, tag_tid=26, entry_tid=52666]]], state=connected] 2026-02-20T21:16:32.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=52669' 2026-02-20T21:16:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=52669' 2026-02-20T21:16:32.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=52669' '!=' 'object_number=363, tag_tid=26, entry_tid=52311' 2026-02-20T21:16:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=52669' = 'object_number=363, tag_tid=26, entry_tid=52311' 2026-02-20T21:16:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=52669' 2026-02-20T21:16:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:32.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=52669' 2026-02-20T21:16:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:32.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:32.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:32.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=52669' 2026-02-20T21:16:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:32.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52669 2026-02-20T21:16:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 52669 2026-02-20T21:16:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:32.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:32.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:32.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:32.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:32.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:32.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:32.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:32.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:32.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:32.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:32.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:32.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:32.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:32.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:32.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:32.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:32.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:32.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:32.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:32.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:32.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:32.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=52989], [object_number=360, tag_tid=26, entry_tid=52988], [object_number=363, tag_tid=26, entry_tid=52987], [object_number=362, tag_tid=26, entry_tid=52986]]], state=connected] 2026-02-20T21:16:32.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:32.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=52989], [object_number=360, tag_tid=26, entry_tid=52988], [object_number=363, tag_tid=26, entry_tid=52987], [object_number=362, tag_tid=26, entry_tid=52986]]], state=connected] 2026-02-20T21:16:32.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=52989' 2026-02-20T21:16:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=52989' 2026-02-20T21:16:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=52989' '!=' 'object_number=361, tag_tid=26, entry_tid=52669' 2026-02-20T21:16:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=52989' = 'object_number=361, tag_tid=26, entry_tid=52669' 2026-02-20T21:16:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=52989' 2026-02-20T21:16:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=52989' 2026-02-20T21:16:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:32.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:32.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=52989' 2026-02-20T21:16:32.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:32.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52989 2026-02-20T21:16:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 52989 2026-02-20T21:16:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:32.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:32.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:32.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:32.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:32.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:32.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:32.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:32.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:32.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:32.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:32.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:32.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:32.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:32.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:32.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:32.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:32.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:32.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:32.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:32.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:32.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:32.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:32.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:32.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:32.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:32.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:32.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:32.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:32.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:32.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:32.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:32.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:32.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:32.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:32.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:32.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=53301], [object_number=360, tag_tid=26, entry_tid=53300], [object_number=363, tag_tid=26, entry_tid=53299], [object_number=362, tag_tid=26, entry_tid=53298]]], state=connected] 2026-02-20T21:16:32.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:32.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=53301], [object_number=360, tag_tid=26, entry_tid=53300], [object_number=363, tag_tid=26, entry_tid=53299], [object_number=362, tag_tid=26, entry_tid=53298]]], state=connected] 2026-02-20T21:16:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=53301' 2026-02-20T21:16:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=53301' 2026-02-20T21:16:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=53301' '!=' 'object_number=361, tag_tid=26, entry_tid=52989' 2026-02-20T21:16:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=53301' = 'object_number=361, tag_tid=26, entry_tid=52989' 2026-02-20T21:16:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=53301' 2026-02-20T21:16:32.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:32.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:32.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:32.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:32.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=53301' 2026-02-20T21:16:32.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:32.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:32.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=53301' 2026-02-20T21:16:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:32.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53301 2026-02-20T21:16:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 53301 2026-02-20T21:16:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:32.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:32.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:32.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:32.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:32.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:32.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:32.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:32.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=53604], [object_number=363, tag_tid=26, entry_tid=53603], [object_number=362, tag_tid=26, entry_tid=53602], [object_number=361, tag_tid=26, entry_tid=53601]]], state=connected] 2026-02-20T21:16:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=53604], [object_number=363, tag_tid=26, entry_tid=53603], [object_number=362, tag_tid=26, entry_tid=53602], [object_number=361, tag_tid=26, entry_tid=53601]]], state=connected] 2026-02-20T21:16:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=360, tag_tid=26, entry_tid=53604' 2026-02-20T21:16:33.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=360, tag_tid=26, entry_tid=53604' 2026-02-20T21:16:33.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=53604' '!=' 'object_number=361, tag_tid=26, entry_tid=53301' 2026-02-20T21:16:33.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:33.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=53604' = 'object_number=361, tag_tid=26, entry_tid=53301' 2026-02-20T21:16:33.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=360, tag_tid=26, entry_tid=53604' 2026-02-20T21:16:33.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:33.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:33.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:33.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:33.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=53604' 2026-02-20T21:16:33.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:33.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:33.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:33.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:33.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:33.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:33.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:33.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=53604' 2026-02-20T21:16:33.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:33.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:33.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53604 2026-02-20T21:16:33.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 53604 2026-02-20T21:16:33.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:33.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:33.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:33.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:33.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:33.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:33.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:33.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:33.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:33.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:33.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:33.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:33.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:33.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:33.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:33.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:33.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=53910], [object_number=361, tag_tid=26, entry_tid=53909], [object_number=360, tag_tid=26, entry_tid=53908], [object_number=363, tag_tid=26, entry_tid=53907]]], state=connected] 2026-02-20T21:16:33.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:33.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:33.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:33.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:33.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:33.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:33.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:33.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=53910], [object_number=361, tag_tid=26, entry_tid=53909], [object_number=360, tag_tid=26, entry_tid=53908], [object_number=363, tag_tid=26, entry_tid=53907]]], state=connected] 2026-02-20T21:16:33.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=362, tag_tid=26, entry_tid=53910' 2026-02-20T21:16:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=362, tag_tid=26, entry_tid=53910' 2026-02-20T21:16:33.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=53910' '!=' 'object_number=360, tag_tid=26, entry_tid=53604' 2026-02-20T21:16:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=53910' = 'object_number=360, tag_tid=26, entry_tid=53604' 2026-02-20T21:16:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=362, tag_tid=26, entry_tid=53910' 2026-02-20T21:16:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:33.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=53910' 2026-02-20T21:16:33.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:33.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:33.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:33.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:33.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:33.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:33.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=53910' 2026-02-20T21:16:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53910 2026-02-20T21:16:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 53910 2026-02-20T21:16:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:33.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:33.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:33.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:33.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:33.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:33.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:33.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:33.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:33.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=54205], [object_number=360, tag_tid=26, entry_tid=54204], [object_number=363, tag_tid=26, entry_tid=54203], [object_number=362, tag_tid=26, entry_tid=54202]]], state=connected] 2026-02-20T21:16:33.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:33.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:33.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:33.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:33.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:33.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:33.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=54205], [object_number=360, tag_tid=26, entry_tid=54204], [object_number=363, tag_tid=26, entry_tid=54203], [object_number=362, tag_tid=26, entry_tid=54202]]], state=connected] 2026-02-20T21:16:33.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=54205' 2026-02-20T21:16:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=54205' 2026-02-20T21:16:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=54205' '!=' 'object_number=362, tag_tid=26, entry_tid=53910' 2026-02-20T21:16:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=54205' = 'object_number=362, tag_tid=26, entry_tid=53910' 2026-02-20T21:16:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=54205' 2026-02-20T21:16:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:33.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:33.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=54205' 2026-02-20T21:16:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:33.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:33.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:33.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=54205' 2026-02-20T21:16:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54205 2026-02-20T21:16:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 54205 2026-02-20T21:16:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:33.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:33.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:33.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:33.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:33.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:33.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:33.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:33.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:34.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:34.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:34.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:34.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=54477], [object_number=360, tag_tid=26, entry_tid=54476], [object_number=363, tag_tid=26, entry_tid=54475], [object_number=362, tag_tid=26, entry_tid=54474]]], state=connected] 2026-02-20T21:16:34.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:34.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:34.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:34.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:34.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:34.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=54477], [object_number=360, tag_tid=26, entry_tid=54476], [object_number=363, tag_tid=26, entry_tid=54475], [object_number=362, tag_tid=26, entry_tid=54474]]], state=connected] 2026-02-20T21:16:34.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=54477' 2026-02-20T21:16:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=54477' 2026-02-20T21:16:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=54477' '!=' 'object_number=361, tag_tid=26, entry_tid=54205' 2026-02-20T21:16:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=54477' = 'object_number=361, tag_tid=26, entry_tid=54205' 2026-02-20T21:16:34.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=54477' 2026-02-20T21:16:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:34.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:34.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=54477' 2026-02-20T21:16:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:34.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:34.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:34.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=54477' 2026-02-20T21:16:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54477 2026-02-20T21:16:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 54477 2026-02-20T21:16:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:34.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:34.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:34.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:34.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:34.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:34.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:34.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:34.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:34.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:34.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:34.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:34.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:34.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:34.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:34.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:34.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:34.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:34.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:34.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:34.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:34.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:34.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:34.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:34.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=54766], [object_number=361, tag_tid=26, entry_tid=54765], [object_number=360, tag_tid=26, entry_tid=54764], [object_number=363, tag_tid=26, entry_tid=54763]]], state=connected] 2026-02-20T21:16:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:34.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:34.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:34.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:34.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:34.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:34.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:34.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:34.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:34.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:34.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=54766], [object_number=361, tag_tid=26, entry_tid=54765], [object_number=360, tag_tid=26, entry_tid=54764], [object_number=363, tag_tid=26, entry_tid=54763]]], state=connected] 2026-02-20T21:16:34.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=362, tag_tid=26, entry_tid=54766' 2026-02-20T21:16:34.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=362, tag_tid=26, entry_tid=54766' 2026-02-20T21:16:34.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=54766' '!=' 'object_number=361, tag_tid=26, entry_tid=54477' 2026-02-20T21:16:34.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:34.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=54766' = 'object_number=361, tag_tid=26, entry_tid=54477' 2026-02-20T21:16:34.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=362, tag_tid=26, entry_tid=54766' 2026-02-20T21:16:34.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:34.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:34.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:34.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:34.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=54766' 2026-02-20T21:16:34.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:34.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:34.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:34.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:34.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:34.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:34.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:34.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=54766' 2026-02-20T21:16:34.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:34.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54766 2026-02-20T21:16:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 54766 2026-02-20T21:16:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:34.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:34.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:34.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:34.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:34.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:34.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:34.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:34.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:34.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:34.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:34.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:34.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:34.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:34.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:34.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:34.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:34.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:34.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:34.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:34.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:34.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:34.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:34.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:34.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:34.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:34.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:34.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=55062], [object_number=361, tag_tid=26, entry_tid=55061], [object_number=360, tag_tid=26, entry_tid=55060], [object_number=363, tag_tid=26, entry_tid=55059]]], state=connected] 2026-02-20T21:16:34.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:34.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:34.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:34.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:34.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:34.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:34.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:34.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:34.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:34.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:34.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:34.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:34.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:34.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:34.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:34.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=362, tag_tid=26, entry_tid=55062], [object_number=361, tag_tid=26, entry_tid=55061], [object_number=360, tag_tid=26, entry_tid=55060], [object_number=363, tag_tid=26, entry_tid=55059]]], state=connected] 2026-02-20T21:16:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=362, tag_tid=26, entry_tid=55062' 2026-02-20T21:16:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=362, tag_tid=26, entry_tid=55062' 2026-02-20T21:16:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=55062' '!=' 'object_number=362, tag_tid=26, entry_tid=54766' 2026-02-20T21:16:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=362, tag_tid=26, entry_tid=55062' = 'object_number=362, tag_tid=26, entry_tid=54766' 2026-02-20T21:16:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=362, tag_tid=26, entry_tid=55062' 2026-02-20T21:16:34.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:34.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=55062' 2026-02-20T21:16:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=362, tag_tid=26, entry_tid=55062' 2026-02-20T21:16:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:34.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55062 2026-02-20T21:16:34.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 55062 2026-02-20T21:16:34.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:34.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:34.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:35.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:35.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:35.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:35.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:35.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:35.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:35.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:35.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=55525], [object_number=360, tag_tid=26, entry_tid=55524], [object_number=363, tag_tid=26, entry_tid=55523], [object_number=362, tag_tid=26, entry_tid=55522]]], state=connected] 2026-02-20T21:16:35.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:35.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:35.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:35.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:35.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:35.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:35.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=361, tag_tid=26, entry_tid=55525], [object_number=360, tag_tid=26, entry_tid=55524], [object_number=363, tag_tid=26, entry_tid=55523], [object_number=362, tag_tid=26, entry_tid=55522]]], state=connected] 2026-02-20T21:16:35.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=361, tag_tid=26, entry_tid=55525' 2026-02-20T21:16:35.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=361, tag_tid=26, entry_tid=55525' 2026-02-20T21:16:35.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=55525' '!=' 'object_number=362, tag_tid=26, entry_tid=55062' 2026-02-20T21:16:35.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:35.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=361, tag_tid=26, entry_tid=55525' = 'object_number=362, tag_tid=26, entry_tid=55062' 2026-02-20T21:16:35.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=361, tag_tid=26, entry_tid=55525' 2026-02-20T21:16:35.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:35.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:35.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=55525' 2026-02-20T21:16:35.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:35.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:35.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:35.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:35.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:35.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:35.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=361, tag_tid=26, entry_tid=55525' 2026-02-20T21:16:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:35.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:35.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55525 2026-02-20T21:16:35.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 55525 2026-02-20T21:16:35.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:35.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:35.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:35.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:35.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:35.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:35.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:35.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:35.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:35.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:35.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:35.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:35.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:35.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:35.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:35.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:35.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:35.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:35.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:35.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:35.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:35.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:35.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:35.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:35.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:35.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:35.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:35.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:35.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:35.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:35.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:35.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:35.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:35.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=55800], [object_number=363, tag_tid=26, entry_tid=55799], [object_number=362, tag_tid=26, entry_tid=55798], [object_number=361, tag_tid=26, entry_tid=55797]]], state=connected] 2026-02-20T21:16:35.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:35.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:35.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:35.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 90 2026-02-20T21:16:35.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:35.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=360, tag_tid=26, entry_tid=55800], [object_number=363, tag_tid=26, entry_tid=55799], [object_number=362, tag_tid=26, entry_tid=55798], [object_number=361, tag_tid=26, entry_tid=55797]]], state=connected] 2026-02-20T21:16:35.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=360, tag_tid=26, entry_tid=55800' 2026-02-20T21:16:35.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=360, tag_tid=26, entry_tid=55800' 2026-02-20T21:16:35.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=55800' '!=' 'object_number=361, tag_tid=26, entry_tid=55525' 2026-02-20T21:16:35.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:35.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=360, tag_tid=26, entry_tid=55800' = 'object_number=361, tag_tid=26, entry_tid=55525' 2026-02-20T21:16:35.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=360, tag_tid=26, entry_tid=55800' 2026-02-20T21:16:35.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:35.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:35.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:35.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:35.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=55800' 2026-02-20T21:16:35.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:35.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:35.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:35.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:35.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:35.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:35.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:35.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=360, tag_tid=26, entry_tid=55800' 2026-02-20T21:16:35.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:35.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:35.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=55800 2026-02-20T21:16:35.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 55800 2026-02-20T21:16:35.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:35.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:35.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:35.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:35.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:35.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:35.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:35.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:35.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:35.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:35.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:35.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:35.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:35.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:35.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=56023], [object_number=366, tag_tid=26, entry_tid=56022], [object_number=365, tag_tid=26, entry_tid=56021], [object_number=364, tag_tid=26, entry_tid=56020]]], state=connected] 2026-02-20T21:16:35.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:35.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=56023], [object_number=366, tag_tid=26, entry_tid=56022], [object_number=365, tag_tid=26, entry_tid=56021], [object_number=364, tag_tid=26, entry_tid=56020]]], state=connected] 2026-02-20T21:16:35.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=56023' 2026-02-20T21:16:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=56023' 2026-02-20T21:16:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=56023' '!=' 'object_number=360, tag_tid=26, entry_tid=55800' 2026-02-20T21:16:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=56023' = 'object_number=360, tag_tid=26, entry_tid=55800' 2026-02-20T21:16:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=56023' 2026-02-20T21:16:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:35.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=56023' 2026-02-20T21:16:35.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:35.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=56023' 2026-02-20T21:16:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:35.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56023 2026-02-20T21:16:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 56023 2026-02-20T21:16:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:35.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:36.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:36.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:36.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:36.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:36.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:36.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:36.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:36.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=56331], [object_number=366, tag_tid=26, entry_tid=56330], [object_number=365, tag_tid=26, entry_tid=56329], [object_number=364, tag_tid=26, entry_tid=56328]]], state=connected] 2026-02-20T21:16:36.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:36.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:36.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:36.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:36.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:36.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:36.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=56331], [object_number=366, tag_tid=26, entry_tid=56330], [object_number=365, tag_tid=26, entry_tid=56329], [object_number=364, tag_tid=26, entry_tid=56328]]], state=connected] 2026-02-20T21:16:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=56331' 2026-02-20T21:16:36.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=56331' 2026-02-20T21:16:36.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=56331' '!=' 'object_number=367, tag_tid=26, entry_tid=56023' 2026-02-20T21:16:36.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:36.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=56331' = 'object_number=367, tag_tid=26, entry_tid=56023' 2026-02-20T21:16:36.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=56331' 2026-02-20T21:16:36.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:36.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:36.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:36.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:36.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=56331' 2026-02-20T21:16:36.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:36.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:36.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=56331' 2026-02-20T21:16:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56331 2026-02-20T21:16:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 56331 2026-02-20T21:16:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:36.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:36.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:36.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:36.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:36.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:36.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:36.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:36.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:36.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:36.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:36.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:36.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:36.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:36.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:36.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:36.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:36.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:36.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:36.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:36.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:36.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:36.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:36.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:36.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:36.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:36.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:36.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:36.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:36.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:36.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:36.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:36.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:36.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:36.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:36.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:36.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:36.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:36.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:36.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=364, tag_tid=26, entry_tid=56684], [object_number=367, tag_tid=26, entry_tid=56683], [object_number=366, tag_tid=26, entry_tid=56682], [object_number=365, tag_tid=26, entry_tid=56681]]], state=connected] 2026-02-20T21:16:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:36.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:36.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:36.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:36.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:36.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:36.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:36.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:36.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:36.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:36.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:36.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:36.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:36.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:36.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:36.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:36.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:36.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:36.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=364, tag_tid=26, entry_tid=56684], [object_number=367, tag_tid=26, entry_tid=56683], [object_number=366, tag_tid=26, entry_tid=56682], [object_number=365, tag_tid=26, entry_tid=56681]]], state=connected] 2026-02-20T21:16:36.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=364, tag_tid=26, entry_tid=56684' 2026-02-20T21:16:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=364, tag_tid=26, entry_tid=56684' 2026-02-20T21:16:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=364, tag_tid=26, entry_tid=56684' '!=' 'object_number=367, tag_tid=26, entry_tid=56331' 2026-02-20T21:16:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=364, tag_tid=26, entry_tid=56684' = 'object_number=367, tag_tid=26, entry_tid=56331' 2026-02-20T21:16:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=364, tag_tid=26, entry_tid=56684' 2026-02-20T21:16:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:36.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:36.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=364, tag_tid=26, entry_tid=56684' 2026-02-20T21:16:36.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:36.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:36.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:36.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:36.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:36.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:36.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:36.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=364, tag_tid=26, entry_tid=56684' 2026-02-20T21:16:36.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:36.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:36.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56684 2026-02-20T21:16:36.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 56684 2026-02-20T21:16:36.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:36.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:36.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:36.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:36.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:36.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:36.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:36.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:36.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:36.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:36.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:36.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:36.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:36.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:36.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:36.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:36.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:36.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:36.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:36.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:36.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:36.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:36.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:36.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:36.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:36.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:36.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:36.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:36.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:36.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:36.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:36.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:36.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:36.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:36.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:36.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:36.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:36.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:36.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:36.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:36.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:36.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:36.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:36.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:36.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:36.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=56939], [object_number=366, tag_tid=26, entry_tid=56938], [object_number=365, tag_tid=26, entry_tid=56937], [object_number=364, tag_tid=26, entry_tid=56936]]], state=connected] 2026-02-20T21:16:36.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:36.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:36.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:36.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:36.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:36.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:36.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:36.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:36.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:36.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:36.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=56939], [object_number=366, tag_tid=26, entry_tid=56938], [object_number=365, tag_tid=26, entry_tid=56937], [object_number=364, tag_tid=26, entry_tid=56936]]], state=connected] 2026-02-20T21:16:36.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=56939' 2026-02-20T21:16:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=56939' 2026-02-20T21:16:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=56939' '!=' 'object_number=364, tag_tid=26, entry_tid=56684' 2026-02-20T21:16:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=56939' = 'object_number=364, tag_tid=26, entry_tid=56684' 2026-02-20T21:16:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=56939' 2026-02-20T21:16:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:36.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=56939' 2026-02-20T21:16:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=56939' 2026-02-20T21:16:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:37.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56939 2026-02-20T21:16:37.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 56939 2026-02-20T21:16:37.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:37.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:37.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:37.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:37.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:37.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:37.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:37.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:37.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:37.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:37.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:37.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:37.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:37.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=57245], [object_number=364, tag_tid=26, entry_tid=57244], [object_number=367, tag_tid=26, entry_tid=57243], [object_number=366, tag_tid=26, entry_tid=57242]]], state=connected] 2026-02-20T21:16:37.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:37.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:37.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:37.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:37.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:37.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:37.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:37.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:37.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=57245], [object_number=364, tag_tid=26, entry_tid=57244], [object_number=367, tag_tid=26, entry_tid=57243], [object_number=366, tag_tid=26, entry_tid=57242]]], state=connected] 2026-02-20T21:16:37.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=57245' 2026-02-20T21:16:37.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=57245' 2026-02-20T21:16:37.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=57245' '!=' 'object_number=367, tag_tid=26, entry_tid=56939' 2026-02-20T21:16:37.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:37.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=57245' = 'object_number=367, tag_tid=26, entry_tid=56939' 2026-02-20T21:16:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=57245' 2026-02-20T21:16:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=57245' 2026-02-20T21:16:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:37.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:37.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=57245' 2026-02-20T21:16:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57245 2026-02-20T21:16:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 57245 2026-02-20T21:16:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:37.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:37.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:37.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:37.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:37.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:37.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:37.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:37.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:37.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:37.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:37.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:37.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:37.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:37.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=57553], [object_number=364, tag_tid=26, entry_tid=57552], [object_number=367, tag_tid=26, entry_tid=57551], [object_number=366, tag_tid=26, entry_tid=57550]]], state=connected] 2026-02-20T21:16:37.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:37.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:37.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:37.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:37.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=57553], [object_number=364, tag_tid=26, entry_tid=57552], [object_number=367, tag_tid=26, entry_tid=57551], [object_number=366, tag_tid=26, entry_tid=57550]]], state=connected] 2026-02-20T21:16:37.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=57553' 2026-02-20T21:16:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=57553' 2026-02-20T21:16:37.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=57553' '!=' 'object_number=365, tag_tid=26, entry_tid=57245' 2026-02-20T21:16:37.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:37.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=57553' = 'object_number=365, tag_tid=26, entry_tid=57245' 2026-02-20T21:16:37.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=57553' 2026-02-20T21:16:37.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:37.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:37.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:37.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:37.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=57553' 2026-02-20T21:16:37.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:37.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:37.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:37.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:37.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=57553' 2026-02-20T21:16:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:37.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:37.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57553 2026-02-20T21:16:37.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 57553 2026-02-20T21:16:37.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:37.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:37.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:37.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:37.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:37.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:37.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:37.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:37.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:37.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:37.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:37.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:37.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:37.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:37.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:37.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:37.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:37.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:37.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:37.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:37.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:37.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:37.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:37.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:37.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:37.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:37.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=57877], [object_number=364, tag_tid=26, entry_tid=57876], [object_number=367, tag_tid=26, entry_tid=57875], [object_number=366, tag_tid=26, entry_tid=57874]]], state=connected] 2026-02-20T21:16:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=57877], [object_number=364, tag_tid=26, entry_tid=57876], [object_number=367, tag_tid=26, entry_tid=57875], [object_number=366, tag_tid=26, entry_tid=57874]]], state=connected] 2026-02-20T21:16:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=57877' 2026-02-20T21:16:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=57877' 2026-02-20T21:16:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=57877' '!=' 'object_number=365, tag_tid=26, entry_tid=57553' 2026-02-20T21:16:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=57877' = 'object_number=365, tag_tid=26, entry_tid=57553' 2026-02-20T21:16:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=57877' 2026-02-20T21:16:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=57877' 2026-02-20T21:16:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:38.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:38.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=57877' 2026-02-20T21:16:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:38.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57877 2026-02-20T21:16:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 57877 2026-02-20T21:16:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:38.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:38.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:38.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:38.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.306 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:16:38.303+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e188f110 2026-02-20T21:16:38.307 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:16:38.305+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb5f0 2026-02-20T21:16:38.308 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:16:38.305+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc8a90 2026-02-20T21:16:38.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:38.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:38.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:38.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:38.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=58147], [object_number=366, tag_tid=26, entry_tid=58146], [object_number=365, tag_tid=26, entry_tid=58145], [object_number=364, tag_tid=26, entry_tid=58144]]], state=connected] 2026-02-20T21:16:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:38.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=58147], [object_number=366, tag_tid=26, entry_tid=58146], [object_number=365, tag_tid=26, entry_tid=58145], [object_number=364, tag_tid=26, entry_tid=58144]]], state=connected] 2026-02-20T21:16:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=58147' 2026-02-20T21:16:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=58147' 2026-02-20T21:16:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=58147' '!=' 'object_number=365, tag_tid=26, entry_tid=57877' 2026-02-20T21:16:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=58147' = 'object_number=365, tag_tid=26, entry_tid=57877' 2026-02-20T21:16:38.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=58147' 2026-02-20T21:16:38.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:38.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:38.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:38.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:38.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=58147' 2026-02-20T21:16:38.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:38.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:38.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:38.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=58147' 2026-02-20T21:16:38.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:38.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58147 2026-02-20T21:16:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 58147 2026-02-20T21:16:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:38.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:38.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:38.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:38.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:38.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:38.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:38.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:38.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:38.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=58441], [object_number=364, tag_tid=26, entry_tid=58440], [object_number=367, tag_tid=26, entry_tid=58439], [object_number=366, tag_tid=26, entry_tid=58438]]], state=connected] 2026-02-20T21:16:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:38.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:38.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:38.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=365, tag_tid=26, entry_tid=58441], [object_number=364, tag_tid=26, entry_tid=58440], [object_number=367, tag_tid=26, entry_tid=58439], [object_number=366, tag_tid=26, entry_tid=58438]]], state=connected] 2026-02-20T21:16:38.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=365, tag_tid=26, entry_tid=58441' 2026-02-20T21:16:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=365, tag_tid=26, entry_tid=58441' 2026-02-20T21:16:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=58441' '!=' 'object_number=367, tag_tid=26, entry_tid=58147' 2026-02-20T21:16:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=365, tag_tid=26, entry_tid=58441' = 'object_number=367, tag_tid=26, entry_tid=58147' 2026-02-20T21:16:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=365, tag_tid=26, entry_tid=58441' 2026-02-20T21:16:38.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:38.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=58441' 2026-02-20T21:16:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:38.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:38.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:38.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:38.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:38.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:38.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:38.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=365, tag_tid=26, entry_tid=58441' 2026-02-20T21:16:38.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:38.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:38.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58441 2026-02-20T21:16:38.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 58441 2026-02-20T21:16:38.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:38.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:38.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:39.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:39.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:39.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:39.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:39.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:39.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:39.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:39.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:39.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:39.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:39.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:39.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:39.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:39.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=58922], [object_number=365, tag_tid=26, entry_tid=58921], [object_number=364, tag_tid=26, entry_tid=58920], [object_number=367, tag_tid=26, entry_tid=58919]]], state=connected] 2026-02-20T21:16:39.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:39.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:39.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:39.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:39.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:39.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:39.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:39.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:39.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:39.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:39.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:39.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:39.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:39.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:39.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=58922], [object_number=365, tag_tid=26, entry_tid=58921], [object_number=364, tag_tid=26, entry_tid=58920], [object_number=367, tag_tid=26, entry_tid=58919]]], state=connected] 2026-02-20T21:16:39.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=366, tag_tid=26, entry_tid=58922' 2026-02-20T21:16:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=366, tag_tid=26, entry_tid=58922' 2026-02-20T21:16:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=58922' '!=' 'object_number=365, tag_tid=26, entry_tid=58441' 2026-02-20T21:16:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=58922' = 'object_number=365, tag_tid=26, entry_tid=58441' 2026-02-20T21:16:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=366, tag_tid=26, entry_tid=58922' 2026-02-20T21:16:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:39.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=58922' 2026-02-20T21:16:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:39.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:39.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:39.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:39.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=58922' 2026-02-20T21:16:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:39.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=58922 2026-02-20T21:16:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 58922 2026-02-20T21:16:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:39.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:39.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:39.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:39.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:39.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:39.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:39.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:39.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:39.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:39.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:39.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:39.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:39.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:39.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:39.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=59222], [object_number=365, tag_tid=26, entry_tid=59221], [object_number=364, tag_tid=26, entry_tid=59220], [object_number=367, tag_tid=26, entry_tid=59219]]], state=connected] 2026-02-20T21:16:39.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:39.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:39.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:39.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:39.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:39.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=59222], [object_number=365, tag_tid=26, entry_tid=59221], [object_number=364, tag_tid=26, entry_tid=59220], [object_number=367, tag_tid=26, entry_tid=59219]]], state=connected] 2026-02-20T21:16:39.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=366, tag_tid=26, entry_tid=59222' 2026-02-20T21:16:39.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=366, tag_tid=26, entry_tid=59222' 2026-02-20T21:16:39.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=59222' '!=' 'object_number=366, tag_tid=26, entry_tid=58922' 2026-02-20T21:16:39.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:39.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=59222' = 'object_number=366, tag_tid=26, entry_tid=58922' 2026-02-20T21:16:39.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=366, tag_tid=26, entry_tid=59222' 2026-02-20T21:16:39.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:39.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:39.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:39.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=59222' 2026-02-20T21:16:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:39.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:39.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:39.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:39.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:39.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=59222' 2026-02-20T21:16:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=59222 2026-02-20T21:16:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 59222 2026-02-20T21:16:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:39.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:39.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:39.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:39.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:39.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:39.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=59523], [object_number=366, tag_tid=26, entry_tid=59522], [object_number=365, tag_tid=26, entry_tid=59521], [object_number=364, tag_tid=26, entry_tid=59520]]], state=connected] 2026-02-20T21:16:39.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:39.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:39.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:39.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:39.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=367, tag_tid=26, entry_tid=59523], [object_number=366, tag_tid=26, entry_tid=59522], [object_number=365, tag_tid=26, entry_tid=59521], [object_number=364, tag_tid=26, entry_tid=59520]]], state=connected] 2026-02-20T21:16:39.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=367, tag_tid=26, entry_tid=59523' 2026-02-20T21:16:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=367, tag_tid=26, entry_tid=59523' 2026-02-20T21:16:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=59523' '!=' 'object_number=366, tag_tid=26, entry_tid=59222' 2026-02-20T21:16:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=367, tag_tid=26, entry_tid=59523' = 'object_number=366, tag_tid=26, entry_tid=59222' 2026-02-20T21:16:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=367, tag_tid=26, entry_tid=59523' 2026-02-20T21:16:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=59523' 2026-02-20T21:16:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:39.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=367, tag_tid=26, entry_tid=59523' 2026-02-20T21:16:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:39.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=59523 2026-02-20T21:16:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 59523 2026-02-20T21:16:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:39.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:40.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:40.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:40.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:40.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:40.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:40.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:40.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:40.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:40.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:40.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:40.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:40.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:40.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:40.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:40.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:40.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:40.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:40.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:40.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:40.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:40.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:40.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=59794], [object_number=365, tag_tid=26, entry_tid=59793], [object_number=364, tag_tid=26, entry_tid=59792], [object_number=367, tag_tid=26, entry_tid=59791]]], state=connected] 2026-02-20T21:16:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:40.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:40.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:40.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 91 2026-02-20T21:16:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:40.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=366, tag_tid=26, entry_tid=59794], [object_number=365, tag_tid=26, entry_tid=59793], [object_number=364, tag_tid=26, entry_tid=59792], [object_number=367, tag_tid=26, entry_tid=59791]]], state=connected] 2026-02-20T21:16:40.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=366, tag_tid=26, entry_tid=59794' 2026-02-20T21:16:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=366, tag_tid=26, entry_tid=59794' 2026-02-20T21:16:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=59794' '!=' 'object_number=367, tag_tid=26, entry_tid=59523' 2026-02-20T21:16:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=366, tag_tid=26, entry_tid=59794' = 'object_number=367, tag_tid=26, entry_tid=59523' 2026-02-20T21:16:40.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=366, tag_tid=26, entry_tid=59794' 2026-02-20T21:16:40.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:40.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:40.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=59794' 2026-02-20T21:16:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:40.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:40.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:40.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:40.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:40.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:40.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=366, tag_tid=26, entry_tid=59794' 2026-02-20T21:16:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:40.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=59794 2026-02-20T21:16:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 59794 2026-02-20T21:16:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:40.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:40.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:40.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:40.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:40.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:40.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:40.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:40.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:40.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:40.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:40.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:40.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:40.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:40.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:40.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:40.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:40.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:40.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:40.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:40.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:40.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:40.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=369, tag_tid=26, entry_tid=60065], [object_number=368, tag_tid=26, entry_tid=60064], [object_number=371, tag_tid=26, entry_tid=60063], [object_number=370, tag_tid=26, entry_tid=60062]]], state=connected] 2026-02-20T21:16:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:40.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:40.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:40.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=369, tag_tid=26, entry_tid=60065], [object_number=368, tag_tid=26, entry_tid=60064], [object_number=371, tag_tid=26, entry_tid=60063], [object_number=370, tag_tid=26, entry_tid=60062]]], state=connected] 2026-02-20T21:16:40.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=369, tag_tid=26, entry_tid=60065' 2026-02-20T21:16:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=369, tag_tid=26, entry_tid=60065' 2026-02-20T21:16:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=369, tag_tid=26, entry_tid=60065' '!=' 'object_number=366, tag_tid=26, entry_tid=59794' 2026-02-20T21:16:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=369, tag_tid=26, entry_tid=60065' = 'object_number=366, tag_tid=26, entry_tid=59794' 2026-02-20T21:16:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=369, tag_tid=26, entry_tid=60065' 2026-02-20T21:16:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=26, entry_tid=60065' 2026-02-20T21:16:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:40.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:40.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:40.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:40.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:40.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:40.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=26, entry_tid=60065' 2026-02-20T21:16:40.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:40.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:40.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=60065 2026-02-20T21:16:40.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 60065 2026-02-20T21:16:40.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:40.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:40.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:40.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:40.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:40.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:40.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:40.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:40.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:40.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:40.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=60378], [object_number=369, tag_tid=26, entry_tid=60377], [object_number=368, tag_tid=26, entry_tid=60376], [object_number=371, tag_tid=26, entry_tid=60375]]], state=connected] 2026-02-20T21:16:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:40.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=60378], [object_number=369, tag_tid=26, entry_tid=60377], [object_number=368, tag_tid=26, entry_tid=60376], [object_number=371, tag_tid=26, entry_tid=60375]]], state=connected] 2026-02-20T21:16:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=370, tag_tid=26, entry_tid=60378' 2026-02-20T21:16:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=370, tag_tid=26, entry_tid=60378' 2026-02-20T21:16:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=60378' '!=' 'object_number=369, tag_tid=26, entry_tid=60065' 2026-02-20T21:16:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=60378' = 'object_number=369, tag_tid=26, entry_tid=60065' 2026-02-20T21:16:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=370, tag_tid=26, entry_tid=60378' 2026-02-20T21:16:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=370, tag_tid=26, entry_tid=60378' 2026-02-20T21:16:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:40.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=370, tag_tid=26, entry_tid=60378' 2026-02-20T21:16:40.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=60378 2026-02-20T21:16:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 60378 2026-02-20T21:16:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:40.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:41.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:41.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:41.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:41.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:41.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:41.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=371, tag_tid=26, entry_tid=60699], [object_number=370, tag_tid=26, entry_tid=60698], [object_number=369, tag_tid=26, entry_tid=60697], [object_number=368, tag_tid=26, entry_tid=60696]]], state=connected] 2026-02-20T21:16:41.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:41.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:41.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:41.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:41.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:41.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:41.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=371, tag_tid=26, entry_tid=60699], [object_number=370, tag_tid=26, entry_tid=60698], [object_number=369, tag_tid=26, entry_tid=60697], [object_number=368, tag_tid=26, entry_tid=60696]]], state=connected] 2026-02-20T21:16:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=371, tag_tid=26, entry_tid=60699' 2026-02-20T21:16:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=371, tag_tid=26, entry_tid=60699' 2026-02-20T21:16:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=371, tag_tid=26, entry_tid=60699' '!=' 'object_number=370, tag_tid=26, entry_tid=60378' 2026-02-20T21:16:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=371, tag_tid=26, entry_tid=60699' = 'object_number=370, tag_tid=26, entry_tid=60378' 2026-02-20T21:16:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=371, tag_tid=26, entry_tid=60699' 2026-02-20T21:16:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=371, tag_tid=26, entry_tid=60699' 2026-02-20T21:16:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:41.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:41.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:41.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=371, tag_tid=26, entry_tid=60699' 2026-02-20T21:16:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:41.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=60699 2026-02-20T21:16:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 60699 2026-02-20T21:16:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:41.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:41.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:41.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:41.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:41.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:41.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:41.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:41.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:41.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:41.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:41.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:41.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=369, tag_tid=26, entry_tid=61021], [object_number=368, tag_tid=26, entry_tid=61020], [object_number=371, tag_tid=26, entry_tid=61019], [object_number=370, tag_tid=26, entry_tid=61018]]], state=connected] 2026-02-20T21:16:41.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:41.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:41.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:41.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:41.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:41.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:41.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:41.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:41.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:41.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:41.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:41.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=369, tag_tid=26, entry_tid=61021], [object_number=368, tag_tid=26, entry_tid=61020], [object_number=371, tag_tid=26, entry_tid=61019], [object_number=370, tag_tid=26, entry_tid=61018]]], state=connected] 2026-02-20T21:16:41.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=369, tag_tid=26, entry_tid=61021' 2026-02-20T21:16:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=369, tag_tid=26, entry_tid=61021' 2026-02-20T21:16:41.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=369, tag_tid=26, entry_tid=61021' '!=' 'object_number=371, tag_tid=26, entry_tid=60699' 2026-02-20T21:16:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=369, tag_tid=26, entry_tid=61021' = 'object_number=371, tag_tid=26, entry_tid=60699' 2026-02-20T21:16:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=369, tag_tid=26, entry_tid=61021' 2026-02-20T21:16:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:41.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:41.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=26, entry_tid=61021' 2026-02-20T21:16:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:41.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:41.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:41.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:41.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:41.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:41.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=26, entry_tid=61021' 2026-02-20T21:16:41.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:41.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:41.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=61021 2026-02-20T21:16:41.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 61021 2026-02-20T21:16:41.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:41.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:41.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:41.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:41.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:41.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:41.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:41.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:41.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:41.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:41.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:41.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:41.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:41.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:41.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:41.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:41.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:41.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:41.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=61326], [object_number=369, tag_tid=26, entry_tid=61325], [object_number=368, tag_tid=26, entry_tid=61324], [object_number=371, tag_tid=26, entry_tid=61323]]], state=connected] 2026-02-20T21:16:41.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:41.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:41.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:41.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:42.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:42.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:42.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:42.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=61326], [object_number=369, tag_tid=26, entry_tid=61325], [object_number=368, tag_tid=26, entry_tid=61324], [object_number=371, tag_tid=26, entry_tid=61323]]], state=connected] 2026-02-20T21:16:42.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=370, tag_tid=26, entry_tid=61326' 2026-02-20T21:16:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=370, tag_tid=26, entry_tid=61326' 2026-02-20T21:16:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=61326' '!=' 'object_number=369, tag_tid=26, entry_tid=61021' 2026-02-20T21:16:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=61326' = 'object_number=369, tag_tid=26, entry_tid=61021' 2026-02-20T21:16:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=370, tag_tid=26, entry_tid=61326' 2026-02-20T21:16:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=370, tag_tid=26, entry_tid=61326' 2026-02-20T21:16:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=370, tag_tid=26, entry_tid=61326' 2026-02-20T21:16:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:42.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=61326 2026-02-20T21:16:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 61326 2026-02-20T21:16:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:42.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:42.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:42.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:42.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:42.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:42.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:42.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:42.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:42.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=61594], [object_number=369, tag_tid=26, entry_tid=61593], [object_number=368, tag_tid=26, entry_tid=61592], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:42.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:42.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:42.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:42.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:42.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:42.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:42.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:42.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=61594], [object_number=369, tag_tid=26, entry_tid=61593], [object_number=368, tag_tid=26, entry_tid=61592], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:42.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:42.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:42.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=61594' '!=' 'object_number=370, tag_tid=26, entry_tid=61326' 2026-02-20T21:16:42.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:16:42.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=61594' = 'object_number=370, tag_tid=26, entry_tid=61326' 2026-02-20T21:16:42.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=27 2026-02-20T21:16:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:16:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:42.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=26 2026-02-20T21:16:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:42.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:16:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:16:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:16:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=61594 2026-02-20T21:16:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 27 = 26 -a 1 -le 61594 2026-02-20T21:16:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:16:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:42.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:42.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:42.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:42.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:42.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:42.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:42.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:42.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:42.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:42.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:42.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:42.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:42.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:42.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:42.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:42.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:42.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:42.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:42.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:42.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=61594], [object_number=369, tag_tid=26, entry_tid=61593], [object_number=368, tag_tid=26, entry_tid=61592], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:42.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:42.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:42.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:42.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=61594], [object_number=369, tag_tid=26, entry_tid=61593], [object_number=368, tag_tid=26, entry_tid=61592], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:42.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=61594' '!=' 'object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:16:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:43.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:43.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:43.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:43.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:43.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:43.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:43.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:43.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:43.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:43.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:43.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:43.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:43.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:43.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:43.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:43.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:43.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:43.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:43.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:43.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:43.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:43.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:43.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:43.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:43.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=61594], [object_number=369, tag_tid=26, entry_tid=61593], [object_number=368, tag_tid=26, entry_tid=61592], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:43.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:43.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:43.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:43.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:43.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:43.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:43.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:43.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:43.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:43.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=61594], [object_number=369, tag_tid=26, entry_tid=61593], [object_number=368, tag_tid=26, entry_tid=61592], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:43.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:43.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:43.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=61594' '!=' 'object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:43.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:16:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:44.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:44.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:44.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:44.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:44.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:44.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:44.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:44.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:44.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:44.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=61594], [object_number=369, tag_tid=26, entry_tid=61593], [object_number=368, tag_tid=26, entry_tid=61592], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=61594], [object_number=369, tag_tid=26, entry_tid=61593], [object_number=368, tag_tid=26, entry_tid=61592], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:44.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=61594' '!=' 'object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:16:45.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:45.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:45.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:45.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:45.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:45.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:45.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:45.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:45.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:45.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:45.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:45.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:45.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:45.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:45.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:45.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:45.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:45.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:45.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:45.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:46.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:46.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:46.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:46.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:46.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:46.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:46.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:46.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:46.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:46.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:46.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:46.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:46.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:46.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:46.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:46.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:46.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=61594], [object_number=369, tag_tid=26, entry_tid=61593], [object_number=368, tag_tid=26, entry_tid=61592], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:46.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:46.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:46.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:46.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:46.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:46.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:46.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:46.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:46.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:46.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=370, tag_tid=26, entry_tid=61594], [object_number=369, tag_tid=26, entry_tid=61593], [object_number=368, tag_tid=26, entry_tid=61592], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:46.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:46.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:46.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:46.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=370, tag_tid=26, entry_tid=61594' '!=' 'object_number=370, tag_tid=26, entry_tid=61594' 2026-02-20T21:16:46.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:16:46.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:16:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:16:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:16:48.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:16:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:16:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:16:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:16:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:16:48.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:48.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:48.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:16:48.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:16:48.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:48.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:16:48.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:48.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:48.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:48.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:16:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:16:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:48.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:48.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:48.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:48.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:48.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:48.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:16:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:16:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:16:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:16:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:16:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:16:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:16:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:16:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:16:48.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:48.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:16:48.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 92 2026-02-20T21:16:48.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 92 2026-02-20T21:16:48.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:16:48.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:48.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=369, tag_tid=27, entry_tid=1], [object_number=368, tag_tid=27, entry_tid=0], [object_number=370, tag_tid=26, entry_tid=61594], [object_number=371, tag_tid=26, entry_tid=61591]]], state=connected] 2026-02-20T21:16:48.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:16:48.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:48.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=369, tag_tid=27, entry_tid=1' -a 'object_number=369, tag_tid=27, entry_tid=1' = 'object_number=369, tag_tid=27, entry_tid=1' 2026-02-20T21:16:48.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap9 2026-02-20T21:16:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:16:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9 2026-02-20T21:16:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:16:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:16:48.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:16:49.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap9 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 17 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ba958b175d 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ba958b175d 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:06:40 2026 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:14:31 2026 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:16:40 2026 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ba958b175d 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:16:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:16:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap8-clean ']' 2026-02-20T21:16:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap8-clean 2026-02-20T21:16:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8-clean 2026-02-20T21:16:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:16:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:16:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:16:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:16:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:16:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:16:49.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:16:49.675 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.672+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:49.677 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.674+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.678 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.674+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.678 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.674+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.678 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.674+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.678 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.675+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.832 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.829+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:16:49.836 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.831+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.836 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.832+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.836 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.832+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.836 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.832+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.836 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.833+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.869 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.866+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.869 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.866+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.869 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.866+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.869 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.867+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.869 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.867+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.934 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.931+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:49.934 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.931+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:16:49.934 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.931+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:49.935 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.931+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:49.935 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.931+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:49.935 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.931+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:49.972 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.969+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:16:49.973 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.970+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:16:49.973 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.971+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:49.974 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.971+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:49.975 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.972+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:49.975 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:49.972+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:16:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:16:50.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:16:50.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:50.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap9 2026-02-20T21:16:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9 2026-02-20T21:16:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:16:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:16:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:16:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:16:50.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:16:50.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:16:50.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:16:50.809 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.806+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:16:50.810 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.807+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:16:50.810 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.807+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:50.811 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.808+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:50.811 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.809+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:50.888 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.885+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:16:50.888 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.885+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:16:50.888 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.885+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:16:50.888 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.886+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:16:50.960 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.958+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:16:50.963 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.960+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:50.963 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.960+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:50.963 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.960+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:50.963 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.960+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:50.964 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.960+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:50.964 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.961+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:50.997 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.995+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:50.998 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.995+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:50.998 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.996+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:50.999 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.996+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:50.999 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:50.996+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:16:51.035 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:51.032+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:16:51.035 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:51.032+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:16:51.035 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:51.032+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:16:51.036 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:51.033+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:51.036 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:51.033+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:51.069 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:51.066+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:16:51.069 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:51.066+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:16:51.069 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:51.066+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:16:51.070 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:51.067+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:51.070 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:16:51.068+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:16:51.181 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:16:51.179+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:16:51.182 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:16:51.179+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:16:51.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:16:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:16:51.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:16:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:16:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap9-clean 2026-02-20T21:16:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap9-clean 2026-02-20T21:16:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:16:51.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:51.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:51.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9-clean 2026-02-20T21:16:51.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap9-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:16:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:16:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror test 2026-02-20T21:16:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:16:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:16:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:16:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:16:51.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-20T21:16:51.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:16:51.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror test --debug-rbd=20 --debug-journaler=20 2026-02-20T21:16:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 60700672 bytes 2026-02-20T21:16:53.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 82900992 bytes 2026-02-20T21:16:53.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 132793344 bytes 2026-02-20T21:16:54.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 163579904 bytes 2026-02-20T21:16:54.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 186876928 bytes 2026-02-20T21:16:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 303646720 bytes 2026-02-20T21:16:57.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 378382336 bytes 2026-02-20T21:16:57.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 404641792 bytes 2026-02-20T21:16:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 425932800 bytes 2026-02-20T21:16:59.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 536220672 bytes 2026-02-20T21:17:01.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 649248768 bytes 2026-02-20T21:17:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:17:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:17:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:17:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-20T21:17:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:17:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:17:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:17:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-20T21:17:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local description_pattern= 2026-02-20T21:17:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local service_pattern= 2026-02-20T21:17:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:17:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:17:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-20T21:17:02.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='test: 2026-02-20T21:17:02.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: global_id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:17:02.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+replaying 2026-02-20T21:17:02.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: replaying, {"bytes_per_second":13960196.13,"entries_behind_primary":3154,"entries_per_second":866.63,"non_primary_position":{"entry_tid":58441,"object_number":365,"tag_tid":26},"primary_position":{"entry_tid":1,"object_number":369,"tag_tid":27},"seconds_until_synced":3} 2026-02-20T21:17:02.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: service: mirror.0 on trial163 2026-02-20T21:17:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:16:38 2026-02-20T21:17:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: peer_sites: 2026-02-20T21:17:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: name: cluster2 2026-02-20T21:17:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: state: up+stopped 2026-02-20T21:17:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: description: local image is primary 2026-02-20T21:17:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: last_update: 2026-02-20 21:16:38' 2026-02-20T21:17:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ state: .*up+replaying' 2026-02-20T21:17:02.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: state: up+replaying 2026-02-20T21:17:02.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^ description: .*' 2026-02-20T21:17:02.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: description: replaying, {"bytes_per_second":13960196.13,"entries_behind_primary":3154,"entries_per_second":866.63,"non_primary_position":{"entry_tid":58441,"object_number":365,"tag_tid":26},"primary_position":{"entry_tid":1,"object_number":369,"tag_tid":27},"seconds_until_synced":3} 2026-02-20T21:17:02.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:17:02.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:17:02.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep '^up+' 2026-02-20T21:17:02.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:17:02.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep 'service: *mirror..* on ' 2026-02-20T21:17:02.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: service: mirror.0 on trial163 2026-02-20T21:17:02.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update 2026-02-20T21:17:02.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-20T21:17:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:16:38' 2026-02-20T21:17:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-20 21:16:38' 2026-02-20T21:17:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state_pattern=up+replaying 2026-02-20T21:17:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'prev_last_update=2026-02-20 21:16:38' 2026-02-20T21:17:02.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status 2026-02-20T21:17:02.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ CEPH_ARGS= 2026-02-20T21:17:02.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-20T21:17:02.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status='OKOKOK142044287mirror.0trial16320.2.0-677-gf7870454trueOK43304332mirror.2trial16320.2.0-677-gf7870454falseOK43484351mirror.3trial16320.2.0-677-gf7870454falseOK45004503mirror.1trial16320.2.0-677-gf7870454falseOKtestba923525-490d-4a76-8e5f-b742509a94daup+replayingreplaying, {"bytes_per_second":13960196.13,"entries_behind_primary":3154,"entries_per_second":866.63,"non_primary_position":{"entry_tid":58441,"object_number":365,"tag_tid":26},"primary_position":{"entry_tid":1,"object_number":369,"tag_tid":27},"seconds_until_synced":3}42044287mirror.0trial1632026-02-20 21:16:38cluster28b968eaf-6c24-469f-bea5-c8acb780219eup+stoppedlocal image is primary2026-02-20 21:16:38' 2026-02-20T21:17:02.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_update state 2026-02-20T21:17:02.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-20T21:17:02.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_update='2026-02-20 21:16:38' 2026-02-20T21:17:02.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-20T21:17:02.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=up+replaying 2026-02-20T21:17:02.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo up+replaying 2026-02-20T21:17:02.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep up+replaying 2026-02-20T21:17:02.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:up+replaying 2026-02-20T21:17:02.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:02.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:02.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap10 2026-02-20T21:17:02.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap10 2026-02-20T21:17:02.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:17:02.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:02.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:02.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10 2026-02-20T21:17:02.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:17:04.978 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.43387137361078976) > weight (0.25) 2026-02-20T21:17:04.978 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3276188499355721) > weight (0.25) 2026-02-20T21:17:04.978 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-20T21:17:04.979 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-20T21:17:04.979 INFO:tasks.rbd_mirror_thrash:waiting for 9.598868459715884 secs before reviving daemons 2026-02-20T21:17:04.979 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:17:04.978+0000 7ff00de29640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 47349) UID: 1000 2026-02-20T21:17:05.057 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-20T21:17:05.058 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-20T21:17:05.058 INFO:tasks.rbd_mirror_thrash:waiting for 0.8386918551460698 secs before reviving daemons 2026-02-20T21:17:05.058 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:17:05.057+0000 7f506f1f1640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 47092) UID: 1000 2026-02-20T21:17:05.898 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-20T21:17:05.898 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:17:12.000 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-20T21:17:12.000 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-20T21:17:12.000 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-20T21:17:12.002 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-20T21:17:12.002 INFO:tasks.rbd_mirror_thrash:waiting for 107.79916732588302 secs before thrashing 2026-02-20T21:17:14.578 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-20T21:17:14.578 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:17:14.579 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-20T21:17:14.579 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-20T21:17:14.579 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-20T21:17:14.582 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-20T21:17:14.582 INFO:tasks.rbd_mirror_thrash:waiting for 77.68590749116001 secs before thrashing 2026-02-20T21:17:31.923 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-20T21:17:31.921+0000 7fdf524e2640 -1 bdev(0x55a614e8f400 /var/lib/ceph/osd/cluster2-2/block) aio_submit retries 2 2026-02-20T21:17:31.924 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-20T21:17:31.922+0000 7fdf594f0640 -1 bdev(0x55a614e8f400 /var/lib/ceph/osd/cluster2-2/block) aio_submit retries 2 2026-02-20T21:17:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:17:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:17:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:17:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:17:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:17:35.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:17:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:17:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:17:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:17:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:17:35.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:17:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:17:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:17:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:17:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:17:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:17:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:35.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:17:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:17:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:17:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:17:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:17:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:17:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:17:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:17:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:17:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:17:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:17:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:17:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:17:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:17:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:17:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:17:35.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:17:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:17:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:17:35.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:35.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:17:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:36.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:36.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:36.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:36.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:17:36.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:17:36.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:17:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:17:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:17:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:17:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:17:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:17:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:17:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:17:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:36.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:17:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:17:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:17:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:17:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:17:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:17:36.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:36.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:36.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:36.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:36.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:17:36.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:17:36.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:17:36.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:17:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:17:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:17:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:17:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:17:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:17:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:17:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:17:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:17:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:17:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:36.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:17:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:17:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:17:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:17:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:17:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:17:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:17:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:17:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:36.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:36.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:36.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:36.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:36.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:36.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:36.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:36.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:36.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:36.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:36.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:36.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:36.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:36.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:36.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:36.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:36.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:36.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:36.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:36.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:36.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:36.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:36.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:36.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:36.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:36.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:36.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:36.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:36.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:36.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:36.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:36.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:36.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:36.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:36.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:36.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:36.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:36.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:36.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:36.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:36.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:36.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 99 2026-02-20T21:17:36.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:36.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:36.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:36.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=397, tag_tid=29, entry_tid=29865], [object_number=396, tag_tid=29, entry_tid=29864], [object_number=399, tag_tid=29, entry_tid=29863], [object_number=398, tag_tid=29, entry_tid=29862]]], state=connected] 2026-02-20T21:17:36.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:36.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:36.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:36.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:36.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:36.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 99 2026-02-20T21:17:36.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:36.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:36.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:36.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=397, tag_tid=29, entry_tid=29865], [object_number=396, tag_tid=29, entry_tid=29864], [object_number=399, tag_tid=29, entry_tid=29863], [object_number=398, tag_tid=29, entry_tid=29862]]], state=connected] 2026-02-20T21:17:36.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:36.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=397, tag_tid=29, entry_tid=29865' 2026-02-20T21:17:36.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=397, tag_tid=29, entry_tid=29865' 2026-02-20T21:17:36.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=397, tag_tid=29, entry_tid=29865' '!=' '' 2026-02-20T21:17:36.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:36.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=397, tag_tid=29, entry_tid=29865' = '' 2026-02-20T21:17:36.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=397, tag_tid=29, entry_tid=29865' 2026-02-20T21:17:36.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:36.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:36.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:36.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:36.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=397, tag_tid=29, entry_tid=29865' 2026-02-20T21:17:36.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:36.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:36.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:36.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:36.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:36.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:36.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:36.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=397, tag_tid=29, entry_tid=29865' 2026-02-20T21:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29865 2026-02-20T21:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 29865 2026-02-20T21:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:36.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:36.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:36.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:36.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:36.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:36.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:36.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:36.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:36.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:36.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:36.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:36.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:36.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:36.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:36.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:36.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:36.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:36.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:36.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:36.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:36.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:36.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:36.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:36.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:36.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:36.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:36.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:36.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:36.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:36.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:36.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:36.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:36.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:36.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:36.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:36.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:36.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:36.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:36.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:36.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:36.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:36.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 99 2026-02-20T21:17:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=396, tag_tid=29, entry_tid=30308], [object_number=399, tag_tid=29, entry_tid=30307], [object_number=398, tag_tid=29, entry_tid=30306], [object_number=397, tag_tid=29, entry_tid=30305]]], state=connected] 2026-02-20T21:17:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:36.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:36.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:36.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 99 2026-02-20T21:17:37.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:37.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:37.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:37.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=396, tag_tid=29, entry_tid=30308], [object_number=399, tag_tid=29, entry_tid=30307], [object_number=398, tag_tid=29, entry_tid=30306], [object_number=397, tag_tid=29, entry_tid=30305]]], state=connected] 2026-02-20T21:17:37.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=396, tag_tid=29, entry_tid=30308' 2026-02-20T21:17:37.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=396, tag_tid=29, entry_tid=30308' 2026-02-20T21:17:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=396, tag_tid=29, entry_tid=30308' '!=' 'object_number=397, tag_tid=29, entry_tid=29865' 2026-02-20T21:17:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=396, tag_tid=29, entry_tid=30308' = 'object_number=397, tag_tid=29, entry_tid=29865' 2026-02-20T21:17:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=396, tag_tid=29, entry_tid=30308' 2026-02-20T21:17:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:37.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=396, tag_tid=29, entry_tid=30308' 2026-02-20T21:17:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:37.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:37.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:37.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:37.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:37.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:37.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=396, tag_tid=29, entry_tid=30308' 2026-02-20T21:17:37.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:37.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:37.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30308 2026-02-20T21:17:37.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 30308 2026-02-20T21:17:37.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:37.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:37.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:37.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:37.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:37.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:37.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:37.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:37.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:37.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:37.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:37.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:37.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:37.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:37.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:37.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:37.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:37.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:37.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:37.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:37.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:37.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:37.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:37.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:37.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:37.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:37.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 99 2026-02-20T21:17:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:37.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=397, tag_tid=29, entry_tid=30725], [object_number=396, tag_tid=29, entry_tid=30724], [object_number=399, tag_tid=29, entry_tid=30723], [object_number=398, tag_tid=29, entry_tid=30722]]], state=connected] 2026-02-20T21:17:37.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:37.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:37.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:37.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:37.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 99 2026-02-20T21:17:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=397, tag_tid=29, entry_tid=30725], [object_number=396, tag_tid=29, entry_tid=30724], [object_number=399, tag_tid=29, entry_tid=30723], [object_number=398, tag_tid=29, entry_tid=30722]]], state=connected] 2026-02-20T21:17:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=397, tag_tid=29, entry_tid=30725' 2026-02-20T21:17:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=397, tag_tid=29, entry_tid=30725' 2026-02-20T21:17:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=397, tag_tid=29, entry_tid=30725' '!=' 'object_number=396, tag_tid=29, entry_tid=30308' 2026-02-20T21:17:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=397, tag_tid=29, entry_tid=30725' = 'object_number=396, tag_tid=29, entry_tid=30308' 2026-02-20T21:17:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=397, tag_tid=29, entry_tid=30725' 2026-02-20T21:17:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=397, tag_tid=29, entry_tid=30725' 2026-02-20T21:17:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:37.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:37.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:37.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:37.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=397, tag_tid=29, entry_tid=30725' 2026-02-20T21:17:37.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:37.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30725 2026-02-20T21:17:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 30725 2026-02-20T21:17:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:37.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:37.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:37.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:37.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:37.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:37.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:37.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:37.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:37.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:37.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:37.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:37.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:37.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:37.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:37.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:37.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:37.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:37.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:37.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:37.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:37.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:37.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:37.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:37.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:37.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:37.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:37.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:37.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:37.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:37.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:37.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:37.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:37.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:37.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:37.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:37.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:37.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:37.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:37.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 99 2026-02-20T21:17:37.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:37.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:37.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:37.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=399, tag_tid=29, entry_tid=31131], [object_number=398, tag_tid=29, entry_tid=31130], [object_number=397, tag_tid=29, entry_tid=31129], [object_number=396, tag_tid=29, entry_tid=31128]]], state=connected] 2026-02-20T21:17:37.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:37.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:37.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:37.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:37.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:37.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:37.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:37.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:37.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:37.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:37.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:37.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:37.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 99 2026-02-20T21:17:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=399, tag_tid=29, entry_tid=31131], [object_number=398, tag_tid=29, entry_tid=31130], [object_number=397, tag_tid=29, entry_tid=31129], [object_number=396, tag_tid=29, entry_tid=31128]]], state=connected] 2026-02-20T21:17:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=399, tag_tid=29, entry_tid=31131' 2026-02-20T21:17:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=399, tag_tid=29, entry_tid=31131' 2026-02-20T21:17:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=399, tag_tid=29, entry_tid=31131' '!=' 'object_number=397, tag_tid=29, entry_tid=30725' 2026-02-20T21:17:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=399, tag_tid=29, entry_tid=31131' = 'object_number=397, tag_tid=29, entry_tid=30725' 2026-02-20T21:17:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=399, tag_tid=29, entry_tid=31131' 2026-02-20T21:17:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=399, tag_tid=29, entry_tid=31131' 2026-02-20T21:17:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:37.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:37.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=399, tag_tid=29, entry_tid=31131' 2026-02-20T21:17:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:37.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31131 2026-02-20T21:17:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 31131 2026-02-20T21:17:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:37.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:37.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:38.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:38.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:38.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:38.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:38.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:38.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:38.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:38.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:38.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:38.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:38.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:38.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:38.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:38.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:38.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:38.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:38.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:38.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:38.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:38.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:38.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:38.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=403, tag_tid=29, entry_tid=31463], [object_number=402, tag_tid=29, entry_tid=31462], [object_number=401, tag_tid=29, entry_tid=31461], [object_number=400, tag_tid=29, entry_tid=31460]]], state=connected] 2026-02-20T21:17:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=403, tag_tid=29, entry_tid=31463], [object_number=402, tag_tid=29, entry_tid=31462], [object_number=401, tag_tid=29, entry_tid=31461], [object_number=400, tag_tid=29, entry_tid=31460]]], state=connected] 2026-02-20T21:17:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=403, tag_tid=29, entry_tid=31463' 2026-02-20T21:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=403, tag_tid=29, entry_tid=31463' 2026-02-20T21:17:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=403, tag_tid=29, entry_tid=31463' '!=' 'object_number=399, tag_tid=29, entry_tid=31131' 2026-02-20T21:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=403, tag_tid=29, entry_tid=31463' = 'object_number=399, tag_tid=29, entry_tid=31131' 2026-02-20T21:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=403, tag_tid=29, entry_tid=31463' 2026-02-20T21:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:38.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:38.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=403, tag_tid=29, entry_tid=31463' 2026-02-20T21:17:38.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:38.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:38.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:38.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=403, tag_tid=29, entry_tid=31463' 2026-02-20T21:17:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:38.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:38.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31463 2026-02-20T21:17:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 31463 2026-02-20T21:17:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:38.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:38.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:38.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:38.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:38.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:38.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:38.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:38.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:38.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:38.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:38.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:38.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:38.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:38.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:38.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:38.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:38.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:38.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:38.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:38.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:38.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:38.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:38.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:38.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:38.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:38.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=402, tag_tid=29, entry_tid=31870], [object_number=401, tag_tid=29, entry_tid=31869], [object_number=400, tag_tid=29, entry_tid=31868], [object_number=403, tag_tid=29, entry_tid=31867]]], state=connected] 2026-02-20T21:17:38.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:38.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:38.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=402, tag_tid=29, entry_tid=31870], [object_number=401, tag_tid=29, entry_tid=31869], [object_number=400, tag_tid=29, entry_tid=31868], [object_number=403, tag_tid=29, entry_tid=31867]]], state=connected] 2026-02-20T21:17:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=402, tag_tid=29, entry_tid=31870' 2026-02-20T21:17:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=402, tag_tid=29, entry_tid=31870' 2026-02-20T21:17:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=402, tag_tid=29, entry_tid=31870' '!=' 'object_number=403, tag_tid=29, entry_tid=31463' 2026-02-20T21:17:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=402, tag_tid=29, entry_tid=31870' = 'object_number=403, tag_tid=29, entry_tid=31463' 2026-02-20T21:17:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=402, tag_tid=29, entry_tid=31870' 2026-02-20T21:17:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:38.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:38.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=402, tag_tid=29, entry_tid=31870' 2026-02-20T21:17:38.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=402, tag_tid=29, entry_tid=31870' 2026-02-20T21:17:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31870 2026-02-20T21:17:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 31870 2026-02-20T21:17:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:38.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:39.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:39.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:39.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:39.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:39.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:39.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:39.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:39.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:39.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:39.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:39.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:39.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:39.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:39.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:39.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:39.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:39.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:39.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:39.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:39.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:39.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:39.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:39.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:39.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:39.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:39.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:39.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:39.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:39.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:39.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:39.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:39.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:39.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:39.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:39.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:39.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:39.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:39.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:39.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:39.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:39.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:39.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:39.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:39.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:39.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:39.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:39.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=401, tag_tid=29, entry_tid=32337], [object_number=400, tag_tid=29, entry_tid=32336], [object_number=403, tag_tid=29, entry_tid=32335], [object_number=402, tag_tid=29, entry_tid=32334]]], state=connected] 2026-02-20T21:17:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:39.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:39.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:39.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=401, tag_tid=29, entry_tid=32337], [object_number=400, tag_tid=29, entry_tid=32336], [object_number=403, tag_tid=29, entry_tid=32335], [object_number=402, tag_tid=29, entry_tid=32334]]], state=connected] 2026-02-20T21:17:39.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=401, tag_tid=29, entry_tid=32337' 2026-02-20T21:17:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=401, tag_tid=29, entry_tid=32337' 2026-02-20T21:17:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=401, tag_tid=29, entry_tid=32337' '!=' 'object_number=402, tag_tid=29, entry_tid=31870' 2026-02-20T21:17:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=401, tag_tid=29, entry_tid=32337' = 'object_number=402, tag_tid=29, entry_tid=31870' 2026-02-20T21:17:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=401, tag_tid=29, entry_tid=32337' 2026-02-20T21:17:39.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:39.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=401, tag_tid=29, entry_tid=32337' 2026-02-20T21:17:39.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:39.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:39.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:39.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:39.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=401, tag_tid=29, entry_tid=32337' 2026-02-20T21:17:39.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:39.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:39.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32337 2026-02-20T21:17:39.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 32337 2026-02-20T21:17:39.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:39.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:39.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:39.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:39.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:39.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:39.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:39.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:39.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:39.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:39.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:39.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:39.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:39.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:39.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:39.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:39.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:39.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:39.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:39.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:39.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:39.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:39.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:39.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:39.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:39.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:39.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:39.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:39.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:39.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:39.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:39.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=403, tag_tid=29, entry_tid=32775], [object_number=402, tag_tid=29, entry_tid=32774], [object_number=401, tag_tid=29, entry_tid=32773], [object_number=400, tag_tid=29, entry_tid=32772]]], state=connected] 2026-02-20T21:17:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=403, tag_tid=29, entry_tid=32775], [object_number=402, tag_tid=29, entry_tid=32774], [object_number=401, tag_tid=29, entry_tid=32773], [object_number=400, tag_tid=29, entry_tid=32772]]], state=connected] 2026-02-20T21:17:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=403, tag_tid=29, entry_tid=32775' 2026-02-20T21:17:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=403, tag_tid=29, entry_tid=32775' 2026-02-20T21:17:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=403, tag_tid=29, entry_tid=32775' '!=' 'object_number=401, tag_tid=29, entry_tid=32337' 2026-02-20T21:17:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=403, tag_tid=29, entry_tid=32775' = 'object_number=401, tag_tid=29, entry_tid=32337' 2026-02-20T21:17:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=403, tag_tid=29, entry_tid=32775' 2026-02-20T21:17:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=403, tag_tid=29, entry_tid=32775' 2026-02-20T21:17:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:39.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:39.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=403, tag_tid=29, entry_tid=32775' 2026-02-20T21:17:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:39.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=32775 2026-02-20T21:17:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 32775 2026-02-20T21:17:39.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:39.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:39.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:39.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:39.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:39.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:39.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:39.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:39.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:39.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:40.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:40.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:40.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:40.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:40.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:40.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:40.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:40.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:40.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:40.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:40.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=401, tag_tid=29, entry_tid=33197], [object_number=400, tag_tid=29, entry_tid=33196], [object_number=403, tag_tid=29, entry_tid=33195], [object_number=402, tag_tid=29, entry_tid=33194]]], state=connected] 2026-02-20T21:17:40.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:40.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:40.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:40.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:40.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:40.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:40.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:40.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=401, tag_tid=29, entry_tid=33197], [object_number=400, tag_tid=29, entry_tid=33196], [object_number=403, tag_tid=29, entry_tid=33195], [object_number=402, tag_tid=29, entry_tid=33194]]], state=connected] 2026-02-20T21:17:40.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:40.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=401, tag_tid=29, entry_tid=33197' 2026-02-20T21:17:40.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=401, tag_tid=29, entry_tid=33197' 2026-02-20T21:17:40.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=401, tag_tid=29, entry_tid=33197' '!=' 'object_number=403, tag_tid=29, entry_tid=32775' 2026-02-20T21:17:40.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:40.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=401, tag_tid=29, entry_tid=33197' = 'object_number=403, tag_tid=29, entry_tid=32775' 2026-02-20T21:17:40.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=401, tag_tid=29, entry_tid=33197' 2026-02-20T21:17:40.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:40.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:40.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:40.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:40.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=401, tag_tid=29, entry_tid=33197' 2026-02-20T21:17:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:40.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=401, tag_tid=29, entry_tid=33197' 2026-02-20T21:17:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:40.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33197 2026-02-20T21:17:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 33197 2026-02-20T21:17:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:40.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:40.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:40.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:40.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:40.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:40.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:40.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:40.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:40.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:40.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:40.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:40.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:40.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:40.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:40.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:40.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:40.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:40.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:40.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:40.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:40.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:40.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=402, tag_tid=29, entry_tid=33618], [object_number=401, tag_tid=29, entry_tid=33617], [object_number=400, tag_tid=29, entry_tid=33616], [object_number=403, tag_tid=29, entry_tid=33615]]], state=connected] 2026-02-20T21:17:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:40.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:40.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:40.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=402, tag_tid=29, entry_tid=33618], [object_number=401, tag_tid=29, entry_tid=33617], [object_number=400, tag_tid=29, entry_tid=33616], [object_number=403, tag_tid=29, entry_tid=33615]]], state=connected] 2026-02-20T21:17:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:40.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=402, tag_tid=29, entry_tid=33618' 2026-02-20T21:17:40.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=402, tag_tid=29, entry_tid=33618' 2026-02-20T21:17:40.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=402, tag_tid=29, entry_tid=33618' '!=' 'object_number=401, tag_tid=29, entry_tid=33197' 2026-02-20T21:17:40.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:40.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=402, tag_tid=29, entry_tid=33618' = 'object_number=401, tag_tid=29, entry_tid=33197' 2026-02-20T21:17:40.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=402, tag_tid=29, entry_tid=33618' 2026-02-20T21:17:40.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=402, tag_tid=29, entry_tid=33618' 2026-02-20T21:17:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:40.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:40.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=402, tag_tid=29, entry_tid=33618' 2026-02-20T21:17:40.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:40.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:40.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=33618 2026-02-20T21:17:40.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 33618 2026-02-20T21:17:40.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:40.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:40.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:40.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:40.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:40.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:40.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:40.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:40.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:40.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:40.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:40.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:40.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:40.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:40.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:40.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:40.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:40.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:40.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:40.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:41.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:41.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=400, tag_tid=29, entry_tid=34040], [object_number=403, tag_tid=29, entry_tid=34039], [object_number=402, tag_tid=29, entry_tid=34038], [object_number=401, tag_tid=29, entry_tid=34037]]], state=connected] 2026-02-20T21:17:41.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:41.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:41.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=400, tag_tid=29, entry_tid=34040], [object_number=403, tag_tid=29, entry_tid=34039], [object_number=402, tag_tid=29, entry_tid=34038], [object_number=401, tag_tid=29, entry_tid=34037]]], state=connected] 2026-02-20T21:17:41.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:41.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=400, tag_tid=29, entry_tid=34040' 2026-02-20T21:17:41.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=400, tag_tid=29, entry_tid=34040' 2026-02-20T21:17:41.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=400, tag_tid=29, entry_tid=34040' '!=' 'object_number=402, tag_tid=29, entry_tid=33618' 2026-02-20T21:17:41.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:41.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=400, tag_tid=29, entry_tid=34040' = 'object_number=402, tag_tid=29, entry_tid=33618' 2026-02-20T21:17:41.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=400, tag_tid=29, entry_tid=34040' 2026-02-20T21:17:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=400, tag_tid=29, entry_tid=34040' 2026-02-20T21:17:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:41.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=400, tag_tid=29, entry_tid=34040' 2026-02-20T21:17:41.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:41.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:41.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34040 2026-02-20T21:17:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 34040 2026-02-20T21:17:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:41.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:41.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:41.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:41.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:41.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:41.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:41.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:41.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:41.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:41.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:41.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:41.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:41.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:41.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:41.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:41.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:41.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:41.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:41.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:41.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:41.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:41.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:41.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:41.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:41.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:41.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:41.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:41.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:41.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:41.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:41.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:41.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:41.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:41.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:41.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:41.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:41.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:41.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:41.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:41.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:41.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=400, tag_tid=29, entry_tid=34476], [object_number=403, tag_tid=29, entry_tid=34475], [object_number=402, tag_tid=29, entry_tid=34474], [object_number=401, tag_tid=29, entry_tid=34473]]], state=connected] 2026-02-20T21:17:41.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:41.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:41.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:41.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:41.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:41.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:41.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=400, tag_tid=29, entry_tid=34476], [object_number=403, tag_tid=29, entry_tid=34475], [object_number=402, tag_tid=29, entry_tid=34474], [object_number=401, tag_tid=29, entry_tid=34473]]], state=connected] 2026-02-20T21:17:41.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=400, tag_tid=29, entry_tid=34476' 2026-02-20T21:17:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=400, tag_tid=29, entry_tid=34476' 2026-02-20T21:17:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=400, tag_tid=29, entry_tid=34476' '!=' 'object_number=400, tag_tid=29, entry_tid=34040' 2026-02-20T21:17:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=400, tag_tid=29, entry_tid=34476' = 'object_number=400, tag_tid=29, entry_tid=34040' 2026-02-20T21:17:41.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=400, tag_tid=29, entry_tid=34476' 2026-02-20T21:17:41.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:41.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:41.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:41.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=400, tag_tid=29, entry_tid=34476' 2026-02-20T21:17:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:41.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:41.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:41.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:41.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=400, tag_tid=29, entry_tid=34476' 2026-02-20T21:17:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:41.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34476 2026-02-20T21:17:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 34476 2026-02-20T21:17:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:41.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:41.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:41.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:41.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:41.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:41.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:41.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:41.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:41.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:41.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:42.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:42.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:42.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:42.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:42.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:42.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:42.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:42.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:42.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:42.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:42.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:42.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:42.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:42.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:42.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:42.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:42.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:42.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:42.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:42.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:42.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=400, tag_tid=29, entry_tid=35132], [object_number=403, tag_tid=29, entry_tid=35131], [object_number=402, tag_tid=29, entry_tid=35130], [object_number=401, tag_tid=29, entry_tid=35129]]], state=connected] 2026-02-20T21:17:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:42.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:42.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:42.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:42.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:42.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:42.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:42.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:42.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=400, tag_tid=29, entry_tid=35132], [object_number=403, tag_tid=29, entry_tid=35131], [object_number=402, tag_tid=29, entry_tid=35130], [object_number=401, tag_tid=29, entry_tid=35129]]], state=connected] 2026-02-20T21:17:42.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=400, tag_tid=29, entry_tid=35132' 2026-02-20T21:17:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=400, tag_tid=29, entry_tid=35132' 2026-02-20T21:17:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=400, tag_tid=29, entry_tid=35132' '!=' 'object_number=400, tag_tid=29, entry_tid=34476' 2026-02-20T21:17:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=400, tag_tid=29, entry_tid=35132' = 'object_number=400, tag_tid=29, entry_tid=34476' 2026-02-20T21:17:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=400, tag_tid=29, entry_tid=35132' 2026-02-20T21:17:42.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:42.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=400, tag_tid=29, entry_tid=35132' 2026-02-20T21:17:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:42.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:42.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:42.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=400, tag_tid=29, entry_tid=35132' 2026-02-20T21:17:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35132 2026-02-20T21:17:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 35132 2026-02-20T21:17:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:42.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:42.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:42.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:42.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:42.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:42.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:42.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:42.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:42.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:42.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:42.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:42.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:42.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:42.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:42.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:42.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:42.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:42.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:42.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:42.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:42.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:42.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:42.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:42.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:42.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:42.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:42.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:42.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:42.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:42.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:42.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:42.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:42.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:42.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:42.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:42.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:42.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:42.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:42.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:42.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:42.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=35499], [object_number=406, tag_tid=29, entry_tid=35498], [object_number=405, tag_tid=29, entry_tid=35497], [object_number=400, tag_tid=29, entry_tid=35496]]], state=connected] 2026-02-20T21:17:42.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:42.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 100 2026-02-20T21:17:42.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:42.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:42.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:42.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=35499], [object_number=406, tag_tid=29, entry_tid=35498], [object_number=405, tag_tid=29, entry_tid=35497], [object_number=400, tag_tid=29, entry_tid=35496]]], state=connected] 2026-02-20T21:17:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=407, tag_tid=29, entry_tid=35499' 2026-02-20T21:17:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=407, tag_tid=29, entry_tid=35499' 2026-02-20T21:17:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=35499' '!=' 'object_number=400, tag_tid=29, entry_tid=35132' 2026-02-20T21:17:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=35499' = 'object_number=400, tag_tid=29, entry_tid=35132' 2026-02-20T21:17:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=407, tag_tid=29, entry_tid=35499' 2026-02-20T21:17:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=35499' 2026-02-20T21:17:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:42.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=35499' 2026-02-20T21:17:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35499 2026-02-20T21:17:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 35499 2026-02-20T21:17:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:42.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:42.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:42.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:42.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:42.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:42.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:42.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:42.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:42.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:42.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:42.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:42.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:42.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:42.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:42.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:42.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:42.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:42.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:42.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:42.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:42.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:42.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:42.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:42.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:42.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:42.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:42.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:42.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:42.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:42.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:42.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:42.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:42.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:42.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:42.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:42.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:42.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:42.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:42.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:42.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:42.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:42.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:42.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:42.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:42.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:42.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:42.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:43.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:43.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:43.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:43.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:43.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:43.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:43.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:43.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:43.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:43.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:43.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:43.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:43.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:43.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:43.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:43.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:43.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:43.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:43.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=35943], [object_number=406, tag_tid=29, entry_tid=35942], [object_number=405, tag_tid=29, entry_tid=35941], [object_number=404, tag_tid=29, entry_tid=35940]]], state=connected] 2026-02-20T21:17:43.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:43.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:43.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:43.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=35943], [object_number=406, tag_tid=29, entry_tid=35942], [object_number=405, tag_tid=29, entry_tid=35941], [object_number=404, tag_tid=29, entry_tid=35940]]], state=connected] 2026-02-20T21:17:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=407, tag_tid=29, entry_tid=35943' 2026-02-20T21:17:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=407, tag_tid=29, entry_tid=35943' 2026-02-20T21:17:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=35943' '!=' 'object_number=407, tag_tid=29, entry_tid=35499' 2026-02-20T21:17:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=35943' = 'object_number=407, tag_tid=29, entry_tid=35499' 2026-02-20T21:17:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=407, tag_tid=29, entry_tid=35943' 2026-02-20T21:17:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:43.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=35943' 2026-02-20T21:17:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:43.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:43.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:43.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:43.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=35943' 2026-02-20T21:17:43.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:43.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35943 2026-02-20T21:17:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 35943 2026-02-20T21:17:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:43.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:43.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:43.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:43.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:43.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:43.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:43.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:43.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:43.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:43.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:43.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:43.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:43.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:43.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:43.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:43.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:43.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:43.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:43.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:43.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:43.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:43.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:43.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:43.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:43.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:43.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:43.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:43.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:43.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:43.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:43.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:43.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:43.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:43.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:43.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:43.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:43.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:43.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:43.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:43.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:43.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:43.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:43.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:43.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:43.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:43.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:43.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:43.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:43.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:43.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:43.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:43.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:43.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:43.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:43.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:43.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:43.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=36343], [object_number=406, tag_tid=29, entry_tid=36342], [object_number=405, tag_tid=29, entry_tid=36341], [object_number=404, tag_tid=29, entry_tid=36340]]], state=connected] 2026-02-20T21:17:43.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:43.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:43.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:43.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:43.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=36343], [object_number=406, tag_tid=29, entry_tid=36342], [object_number=405, tag_tid=29, entry_tid=36341], [object_number=404, tag_tid=29, entry_tid=36340]]], state=connected] 2026-02-20T21:17:43.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=407, tag_tid=29, entry_tid=36343' 2026-02-20T21:17:43.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=407, tag_tid=29, entry_tid=36343' 2026-02-20T21:17:43.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=36343' '!=' 'object_number=407, tag_tid=29, entry_tid=35943' 2026-02-20T21:17:43.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:43.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=36343' = 'object_number=407, tag_tid=29, entry_tid=35943' 2026-02-20T21:17:43.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=407, tag_tid=29, entry_tid=36343' 2026-02-20T21:17:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=36343' 2026-02-20T21:17:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:43.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:43.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:43.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=36343' 2026-02-20T21:17:43.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:43.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36343 2026-02-20T21:17:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 36343 2026-02-20T21:17:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:43.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:43.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:43.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:43.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:43.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:43.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:43.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:43.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:43.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:43.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:43.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:43.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:43.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:43.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:43.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:43.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:43.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:43.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:43.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:43.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:43.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:43.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:43.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=405, tag_tid=29, entry_tid=36753], [object_number=404, tag_tid=29, entry_tid=36752], [object_number=407, tag_tid=29, entry_tid=36751], [object_number=406, tag_tid=29, entry_tid=36750]]], state=connected] 2026-02-20T21:17:43.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:43.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:43.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:43.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:43.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:43.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:43.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:43.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:43.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:43.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:43.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:43.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:43.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:43.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:43.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:43.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=405, tag_tid=29, entry_tid=36753], [object_number=404, tag_tid=29, entry_tid=36752], [object_number=407, tag_tid=29, entry_tid=36751], [object_number=406, tag_tid=29, entry_tid=36750]]], state=connected] 2026-02-20T21:17:43.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=405, tag_tid=29, entry_tid=36753' 2026-02-20T21:17:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=405, tag_tid=29, entry_tid=36753' 2026-02-20T21:17:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=405, tag_tid=29, entry_tid=36753' '!=' 'object_number=407, tag_tid=29, entry_tid=36343' 2026-02-20T21:17:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=405, tag_tid=29, entry_tid=36753' = 'object_number=407, tag_tid=29, entry_tid=36343' 2026-02-20T21:17:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=405, tag_tid=29, entry_tid=36753' 2026-02-20T21:17:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:44.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:44.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:44.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=405, tag_tid=29, entry_tid=36753' 2026-02-20T21:17:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:44.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:44.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:44.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:44.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:44.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:44.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=405, tag_tid=29, entry_tid=36753' 2026-02-20T21:17:44.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:44.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:44.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36753 2026-02-20T21:17:44.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 36753 2026-02-20T21:17:44.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:44.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:44.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:44.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:44.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:44.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:44.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:44.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:44.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:44.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:44.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:44.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:44.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:44.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:44.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:44.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:44.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:44.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:44.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:44.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:44.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:44.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:44.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:44.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:44.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:44.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:44.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:44.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:44.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:44.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:44.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:44.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:44.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:44.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:44.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:44.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:44.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:44.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:44.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:44.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:44.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=37151], [object_number=406, tag_tid=29, entry_tid=37150], [object_number=405, tag_tid=29, entry_tid=37149], [object_number=404, tag_tid=29, entry_tid=37148]]], state=connected] 2026-02-20T21:17:44.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:44.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:44.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:44.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:44.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:44.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:44.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:44.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:44.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:44.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:44.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:44.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=37151], [object_number=406, tag_tid=29, entry_tid=37150], [object_number=405, tag_tid=29, entry_tid=37149], [object_number=404, tag_tid=29, entry_tid=37148]]], state=connected] 2026-02-20T21:17:44.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=407, tag_tid=29, entry_tid=37151' 2026-02-20T21:17:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=407, tag_tid=29, entry_tid=37151' 2026-02-20T21:17:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=37151' '!=' 'object_number=405, tag_tid=29, entry_tid=36753' 2026-02-20T21:17:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=37151' = 'object_number=405, tag_tid=29, entry_tid=36753' 2026-02-20T21:17:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=407, tag_tid=29, entry_tid=37151' 2026-02-20T21:17:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=37151' 2026-02-20T21:17:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:44.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:44.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:44.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:44.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=37151' 2026-02-20T21:17:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37151 2026-02-20T21:17:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 37151 2026-02-20T21:17:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:44.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:44.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:44.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:44.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:44.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:44.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:44.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:44.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:44.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:44.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:44.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:44.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:44.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:44.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:44.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:44.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:44.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:44.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:44.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:44.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:44.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:44.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:44.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:44.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=37571], [object_number=406, tag_tid=29, entry_tid=37570], [object_number=405, tag_tid=29, entry_tid=37569], [object_number=404, tag_tid=29, entry_tid=37568]]], state=connected] 2026-02-20T21:17:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:44.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:44.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:44.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:44.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=407, tag_tid=29, entry_tid=37571], [object_number=406, tag_tid=29, entry_tid=37570], [object_number=405, tag_tid=29, entry_tid=37569], [object_number=404, tag_tid=29, entry_tid=37568]]], state=connected] 2026-02-20T21:17:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=407, tag_tid=29, entry_tid=37571' 2026-02-20T21:17:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=407, tag_tid=29, entry_tid=37571' 2026-02-20T21:17:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=37571' '!=' 'object_number=407, tag_tid=29, entry_tid=37151' 2026-02-20T21:17:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=407, tag_tid=29, entry_tid=37571' = 'object_number=407, tag_tid=29, entry_tid=37151' 2026-02-20T21:17:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=407, tag_tid=29, entry_tid=37571' 2026-02-20T21:17:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=37571' 2026-02-20T21:17:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:44.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:44.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:44.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:44.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:44.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=407, tag_tid=29, entry_tid=37571' 2026-02-20T21:17:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:44.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:44.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37571 2026-02-20T21:17:44.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 37571 2026-02-20T21:17:44.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:44.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:44.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:45.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:45.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:45.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:45.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:45.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:45.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:45.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:45.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:45.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:45.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:45.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:45.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:45.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:45.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:45.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:45.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:45.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:45.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:45.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:45.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:45.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:45.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:45.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:45.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:45.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:45.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:45.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:45.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:45.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:45.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:45.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:45.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:45.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:45.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:45.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:45.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:45.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=404, tag_tid=29, entry_tid=37976], [object_number=407, tag_tid=29, entry_tid=37975], [object_number=406, tag_tid=29, entry_tid=37974], [object_number=405, tag_tid=29, entry_tid=37973]]], state=connected] 2026-02-20T21:17:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:45.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:45.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:45.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:45.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=404, tag_tid=29, entry_tid=37976], [object_number=407, tag_tid=29, entry_tid=37975], [object_number=406, tag_tid=29, entry_tid=37974], [object_number=405, tag_tid=29, entry_tid=37973]]], state=connected] 2026-02-20T21:17:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=404, tag_tid=29, entry_tid=37976' 2026-02-20T21:17:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=404, tag_tid=29, entry_tid=37976' 2026-02-20T21:17:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=404, tag_tid=29, entry_tid=37976' '!=' 'object_number=407, tag_tid=29, entry_tid=37571' 2026-02-20T21:17:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=404, tag_tid=29, entry_tid=37976' = 'object_number=407, tag_tid=29, entry_tid=37571' 2026-02-20T21:17:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=404, tag_tid=29, entry_tid=37976' 2026-02-20T21:17:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:45.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:45.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:45.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=404, tag_tid=29, entry_tid=37976' 2026-02-20T21:17:45.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:45.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:45.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:45.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:45.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:45.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=404, tag_tid=29, entry_tid=37976' 2026-02-20T21:17:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:45.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:45.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37976 2026-02-20T21:17:45.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 37976 2026-02-20T21:17:45.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:45.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:45.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:45.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:45.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:45.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:45.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:45.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:45.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:45.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:45.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:45.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:45.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:45.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:45.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:45.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:45.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:45.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:45.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:45.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:45.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:45.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:45.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:45.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:45.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:45.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:45.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:45.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:45.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:45.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:45.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:45.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:45.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:45.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=404, tag_tid=29, entry_tid=38376], [object_number=407, tag_tid=29, entry_tid=38375], [object_number=406, tag_tid=29, entry_tid=38374], [object_number=405, tag_tid=29, entry_tid=38373]]], state=connected] 2026-02-20T21:17:45.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:45.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:45.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=404, tag_tid=29, entry_tid=38376], [object_number=407, tag_tid=29, entry_tid=38375], [object_number=406, tag_tid=29, entry_tid=38374], [object_number=405, tag_tid=29, entry_tid=38373]]], state=connected] 2026-02-20T21:17:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:45.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=404, tag_tid=29, entry_tid=38376' 2026-02-20T21:17:45.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=404, tag_tid=29, entry_tid=38376' 2026-02-20T21:17:45.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=404, tag_tid=29, entry_tid=38376' '!=' 'object_number=404, tag_tid=29, entry_tid=37976' 2026-02-20T21:17:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=404, tag_tid=29, entry_tid=38376' = 'object_number=404, tag_tid=29, entry_tid=37976' 2026-02-20T21:17:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=404, tag_tid=29, entry_tid=38376' 2026-02-20T21:17:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:45.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=404, tag_tid=29, entry_tid=38376' 2026-02-20T21:17:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:45.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:45.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:45.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:45.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:45.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:45.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:45.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=404, tag_tid=29, entry_tid=38376' 2026-02-20T21:17:45.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:45.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38376 2026-02-20T21:17:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 38376 2026-02-20T21:17:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:46.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:46.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:46.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:46.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:46.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:46.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:46.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:46.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:46.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:46.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:46.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:46.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:46.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:46.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:46.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:46.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:46.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:46.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:46.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:46.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:46.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:46.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:46.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:46.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:46.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:46.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:46.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:46.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:46.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:46.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:46.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:46.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:46.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=405, tag_tid=29, entry_tid=38777], [object_number=404, tag_tid=29, entry_tid=38776], [object_number=407, tag_tid=29, entry_tid=38775], [object_number=406, tag_tid=29, entry_tid=38774]]], state=connected] 2026-02-20T21:17:46.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:46.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:46.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:46.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:46.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:46.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:46.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:46.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:46.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:46.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:46.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:46.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:46.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=405, tag_tid=29, entry_tid=38777], [object_number=404, tag_tid=29, entry_tid=38776], [object_number=407, tag_tid=29, entry_tid=38775], [object_number=406, tag_tid=29, entry_tid=38774]]], state=connected] 2026-02-20T21:17:46.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:46.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=405, tag_tid=29, entry_tid=38777' 2026-02-20T21:17:46.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=405, tag_tid=29, entry_tid=38777' 2026-02-20T21:17:46.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=405, tag_tid=29, entry_tid=38777' '!=' 'object_number=404, tag_tid=29, entry_tid=38376' 2026-02-20T21:17:46.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:46.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=405, tag_tid=29, entry_tid=38777' = 'object_number=404, tag_tid=29, entry_tid=38376' 2026-02-20T21:17:46.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=405, tag_tid=29, entry_tid=38777' 2026-02-20T21:17:46.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:46.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:46.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=405, tag_tid=29, entry_tid=38777' 2026-02-20T21:17:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:46.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:46.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:46.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=405, tag_tid=29, entry_tid=38777' 2026-02-20T21:17:46.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:46.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38777 2026-02-20T21:17:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 38777 2026-02-20T21:17:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:46.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:46.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:46.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:46.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:46.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:46.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:46.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:46.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:46.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:46.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:46.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:46.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:46.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:46.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:46.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:46.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:46.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:46.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:46.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:46.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:46.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:46.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:46.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:46.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:46.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:46.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:46.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:46.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:46.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:46.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:46.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:46.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:46.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:46.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:46.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:46.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:46.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:46.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:46.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:46.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:46.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:46.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:46.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:46.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:46.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:46.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:46.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:46.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:46.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:46.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:46.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:46.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:46.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:46.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:46.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:46.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:46.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=406, tag_tid=29, entry_tid=39194], [object_number=405, tag_tid=29, entry_tid=39193], [object_number=404, tag_tid=29, entry_tid=39192], [object_number=407, tag_tid=29, entry_tid=39191]]], state=connected] 2026-02-20T21:17:46.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:46.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:46.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:46.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:46.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:46.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:46.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:46.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:46.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:46.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:46.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:46.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=406, tag_tid=29, entry_tid=39194], [object_number=405, tag_tid=29, entry_tid=39193], [object_number=404, tag_tid=29, entry_tid=39192], [object_number=407, tag_tid=29, entry_tid=39191]]], state=connected] 2026-02-20T21:17:46.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:46.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=406, tag_tid=29, entry_tid=39194' 2026-02-20T21:17:46.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=406, tag_tid=29, entry_tid=39194' 2026-02-20T21:17:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=406, tag_tid=29, entry_tid=39194' '!=' 'object_number=405, tag_tid=29, entry_tid=38777' 2026-02-20T21:17:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=406, tag_tid=29, entry_tid=39194' = 'object_number=405, tag_tid=29, entry_tid=38777' 2026-02-20T21:17:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=406, tag_tid=29, entry_tid=39194' 2026-02-20T21:17:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=406, tag_tid=29, entry_tid=39194' 2026-02-20T21:17:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:46.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=406, tag_tid=29, entry_tid=39194' 2026-02-20T21:17:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39194 2026-02-20T21:17:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 39194 2026-02-20T21:17:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:46.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:46.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:46.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:46.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:46.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:46.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:46.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:46.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:46.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:46.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:46.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:46.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:46.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:46.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:46.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:46.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:46.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:46.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:46.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:46.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:46.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:47.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:47.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:47.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:47.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=405, tag_tid=29, entry_tid=39601], [object_number=404, tag_tid=29, entry_tid=39600], [object_number=407, tag_tid=29, entry_tid=39599], [object_number=406, tag_tid=29, entry_tid=39598]]], state=connected] 2026-02-20T21:17:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:47.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:47.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:47.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 101 2026-02-20T21:17:47.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:47.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:47.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:47.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=405, tag_tid=29, entry_tid=39601], [object_number=404, tag_tid=29, entry_tid=39600], [object_number=407, tag_tid=29, entry_tid=39599], [object_number=406, tag_tid=29, entry_tid=39598]]], state=connected] 2026-02-20T21:17:47.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=405, tag_tid=29, entry_tid=39601' 2026-02-20T21:17:47.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=405, tag_tid=29, entry_tid=39601' 2026-02-20T21:17:47.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=405, tag_tid=29, entry_tid=39601' '!=' 'object_number=406, tag_tid=29, entry_tid=39194' 2026-02-20T21:17:47.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:47.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=405, tag_tid=29, entry_tid=39601' = 'object_number=406, tag_tid=29, entry_tid=39194' 2026-02-20T21:17:47.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=405, tag_tid=29, entry_tid=39601' 2026-02-20T21:17:47.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:47.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=405, tag_tid=29, entry_tid=39601' 2026-02-20T21:17:47.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:47.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:47.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=405, tag_tid=29, entry_tid=39601' 2026-02-20T21:17:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39601 2026-02-20T21:17:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 39601 2026-02-20T21:17:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:47.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:47.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:47.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:47.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:47.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:47.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:47.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:47.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:47.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:47.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:47.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:47.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:47.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:47.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:47.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:47.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:47.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:47.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:47.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:47.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:47.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:47.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:47.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:47.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:47.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:47.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:47.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:47.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:47.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:47.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:47.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:47.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:47.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:47.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:47.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:47.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:47.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:47.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:47.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:47.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:47.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:47.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:47.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:47.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:47.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:47.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:47.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:47.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:47.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:47.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:47.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:47.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:47.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:47.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:47.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:47.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:47.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:47.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:47.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:47.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:47.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:47.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:47.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:47.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=410, tag_tid=29, entry_tid=39998], [object_number=409, tag_tid=29, entry_tid=39997], [object_number=408, tag_tid=29, entry_tid=39996], [object_number=411, tag_tid=29, entry_tid=39995]]], state=connected] 2026-02-20T21:17:47.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:47.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:47.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:47.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:47.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=410, tag_tid=29, entry_tid=39998], [object_number=409, tag_tid=29, entry_tid=39997], [object_number=408, tag_tid=29, entry_tid=39996], [object_number=411, tag_tid=29, entry_tid=39995]]], state=connected] 2026-02-20T21:17:47.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=410, tag_tid=29, entry_tid=39998' 2026-02-20T21:17:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=410, tag_tid=29, entry_tid=39998' 2026-02-20T21:17:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=410, tag_tid=29, entry_tid=39998' '!=' 'object_number=405, tag_tid=29, entry_tid=39601' 2026-02-20T21:17:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=410, tag_tid=29, entry_tid=39998' = 'object_number=405, tag_tid=29, entry_tid=39601' 2026-02-20T21:17:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=410, tag_tid=29, entry_tid=39998' 2026-02-20T21:17:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=410, tag_tid=29, entry_tid=39998' 2026-02-20T21:17:47.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:47.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=410, tag_tid=29, entry_tid=39998' 2026-02-20T21:17:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:47.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:47.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39998 2026-02-20T21:17:47.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 39998 2026-02-20T21:17:47.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:47.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:47.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:47.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:47.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:47.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:47.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:47.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:47.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:47.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:47.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:47.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:47.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:47.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:47.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:47.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:47.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:47.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:47.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:47.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:47.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:47.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:47.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:47.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:47.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:47.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:47.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:47.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:47.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:47.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:47.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:47.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:47.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:47.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:47.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:47.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:47.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:47.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:47.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:47.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:47.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:47.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:48.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=410, tag_tid=29, entry_tid=40426], [object_number=409, tag_tid=29, entry_tid=40425], [object_number=408, tag_tid=29, entry_tid=40424], [object_number=411, tag_tid=29, entry_tid=40423]]], state=connected] 2026-02-20T21:17:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=410, tag_tid=29, entry_tid=40426], [object_number=409, tag_tid=29, entry_tid=40425], [object_number=408, tag_tid=29, entry_tid=40424], [object_number=411, tag_tid=29, entry_tid=40423]]], state=connected] 2026-02-20T21:17:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=410, tag_tid=29, entry_tid=40426' 2026-02-20T21:17:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=410, tag_tid=29, entry_tid=40426' 2026-02-20T21:17:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=410, tag_tid=29, entry_tid=40426' '!=' 'object_number=410, tag_tid=29, entry_tid=39998' 2026-02-20T21:17:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=410, tag_tid=29, entry_tid=40426' = 'object_number=410, tag_tid=29, entry_tid=39998' 2026-02-20T21:17:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=410, tag_tid=29, entry_tid=40426' 2026-02-20T21:17:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=410, tag_tid=29, entry_tid=40426' 2026-02-20T21:17:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=410, tag_tid=29, entry_tid=40426' 2026-02-20T21:17:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:48.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40426 2026-02-20T21:17:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 40426 2026-02-20T21:17:48.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:48.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:48.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:48.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:48.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:48.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:48.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:48.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:48.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:48.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:48.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:48.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:48.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:48.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:48.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:48.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:48.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:48.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:48.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:48.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:48.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:48.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:48.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:48.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:48.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:48.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:48.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:48.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:48.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:48.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:48.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:48.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:48.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:48.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:48.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:48.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:48.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:48.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:48.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:48.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:48.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:48.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:48.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=29, entry_tid=40833], [object_number=408, tag_tid=29, entry_tid=40832], [object_number=411, tag_tid=29, entry_tid=40831], [object_number=410, tag_tid=29, entry_tid=40830]]], state=connected] 2026-02-20T21:17:48.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:48.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:48.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:48.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:48.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:48.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=29, entry_tid=40833], [object_number=408, tag_tid=29, entry_tid=40832], [object_number=411, tag_tid=29, entry_tid=40831], [object_number=410, tag_tid=29, entry_tid=40830]]], state=connected] 2026-02-20T21:17:48.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=29, entry_tid=40833' 2026-02-20T21:17:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=409, tag_tid=29, entry_tid=40833' 2026-02-20T21:17:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=29, entry_tid=40833' '!=' 'object_number=410, tag_tid=29, entry_tid=40426' 2026-02-20T21:17:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=29, entry_tid=40833' = 'object_number=410, tag_tid=29, entry_tid=40426' 2026-02-20T21:17:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=409, tag_tid=29, entry_tid=40833' 2026-02-20T21:17:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:48.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:48.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:48.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=29, entry_tid=40833' 2026-02-20T21:17:48.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:48.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:48.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:48.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:48.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:48.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=29, entry_tid=40833' 2026-02-20T21:17:48.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:48.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=40833 2026-02-20T21:17:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 40833 2026-02-20T21:17:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:48.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:48.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:48.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:48.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:48.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:48.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:48.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:48.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:48.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:48.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:48.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:48.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:48.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:48.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:48.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:48.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:48.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:48.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:48.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:48.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:48.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:48.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:48.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:48.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:48.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:48.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:48.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:48.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:48.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:48.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:48.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:48.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:48.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:48.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:48.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:48.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:48.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:48.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:48.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:48.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:48.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:48.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=29, entry_tid=41217], [object_number=408, tag_tid=29, entry_tid=41216], [object_number=411, tag_tid=29, entry_tid=41215], [object_number=410, tag_tid=29, entry_tid=41214]]], state=connected] 2026-02-20T21:17:48.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:48.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=29, entry_tid=41217], [object_number=408, tag_tid=29, entry_tid=41216], [object_number=411, tag_tid=29, entry_tid=41215], [object_number=410, tag_tid=29, entry_tid=41214]]], state=connected] 2026-02-20T21:17:48.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=29, entry_tid=41217' 2026-02-20T21:17:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=409, tag_tid=29, entry_tid=41217' 2026-02-20T21:17:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=29, entry_tid=41217' '!=' 'object_number=409, tag_tid=29, entry_tid=40833' 2026-02-20T21:17:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=29, entry_tid=41217' = 'object_number=409, tag_tid=29, entry_tid=40833' 2026-02-20T21:17:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=409, tag_tid=29, entry_tid=41217' 2026-02-20T21:17:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=29, entry_tid=41217' 2026-02-20T21:17:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:48.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:48.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:48.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:48.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:48.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:48.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:48.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=29, entry_tid=41217' 2026-02-20T21:17:48.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:48.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41217 2026-02-20T21:17:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 41217 2026-02-20T21:17:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:49.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:49.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:49.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:49.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:49.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:49.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:49.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:49.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:49.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:49.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:49.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:49.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:49.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:49.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:49.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:49.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:49.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:49.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:49.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:49.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:49.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:49.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:49.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:49.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:49.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:49.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:49.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:49.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:49.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:49.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:49.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:49.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:49.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:49.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:49.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:49.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:49.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:49.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:49.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:49.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:49.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:49.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314], [object_number=409, tag_tid=29, entry_tid=41313], [object_number=408, tag_tid=29, entry_tid=41312]]], state=connected] 2026-02-20T21:17:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:49.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:49.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:49.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:49.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:49.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:49.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:49.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:49.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:49.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:49.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:49.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:49.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314], [object_number=409, tag_tid=29, entry_tid=41313], [object_number=408, tag_tid=29, entry_tid=41312]]], state=connected] 2026-02-20T21:17:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=411, tag_tid=29, entry_tid=41315' 2026-02-20T21:17:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=411, tag_tid=29, entry_tid=41315' 2026-02-20T21:17:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=411, tag_tid=29, entry_tid=41315' '!=' 'object_number=409, tag_tid=29, entry_tid=41217' 2026-02-20T21:17:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=411, tag_tid=29, entry_tid=41315' = 'object_number=409, tag_tid=29, entry_tid=41217' 2026-02-20T21:17:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=411, tag_tid=29, entry_tid=41315' 2026-02-20T21:17:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=30 2026-02-20T21:17:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=411, tag_tid=29, entry_tid=41315' 2026-02-20T21:17:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=29 2026-02-20T21:17:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:49.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:49.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=411, tag_tid=29, entry_tid=41315' 2026-02-20T21:17:49.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:49.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:49.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=41315 2026-02-20T21:17:49.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 30 = 29 -a 1 -le 41315 2026-02-20T21:17:49.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:49.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:49.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:49.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:49.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:49.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:49.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:49.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:49.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:49.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:49.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:49.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:49.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:49.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:49.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:49.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:49.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:49.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:49.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:49.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:49.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:49.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:49.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:49.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:49.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:49.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:49.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:49.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:49.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:49.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:49.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:49.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:49.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:49.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:49.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:49.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:49.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:49.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:49.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:49.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:49.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:49.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:49.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:49.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:49.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:49.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:49.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:49.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:49.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:49.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314], [object_number=409, tag_tid=29, entry_tid=41313], [object_number=408, tag_tid=29, entry_tid=41312]]], state=connected] 2026-02-20T21:17:49.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:49.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:49.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:49.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:49.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:49.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314], [object_number=409, tag_tid=29, entry_tid=41313], [object_number=408, tag_tid=29, entry_tid=41312]]], state=connected] 2026-02-20T21:17:49.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:49.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=411, tag_tid=29, entry_tid=41315' 2026-02-20T21:17:49.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=411, tag_tid=29, entry_tid=41315' 2026-02-20T21:17:49.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=411, tag_tid=29, entry_tid=41315' '!=' 'object_number=411, tag_tid=29, entry_tid=41315' 2026-02-20T21:17:49.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:49.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:17:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:50.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:50.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:50.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:50.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:50.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:50.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:50.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:50.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:50.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:50.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:50.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:50.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:50.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:50.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:50.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:50.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:50.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:50.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:50.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:50.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:50.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:50.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:50.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:50.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:50.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:50.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:50.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:50.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:50.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:50.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:50.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=30, entry_tid=1' -a 'object_number=409, tag_tid=30, entry_tid=1' = 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap10 2026-02-20T21:17:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:50.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10 2026-02-20T21:17:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:17:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:17:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:17:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap10 2026-02-20T21:17:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-20T21:17:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:17:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:17:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 19 2026-02-20T21:17:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ba958b175d 2026-02-20T21:17:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ba958b175d 2026-02-20T21:17:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:17:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:17:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:17:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:17:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:06:40 2026 2026-02-20T21:17:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:16:49 2026 2026-02-20T21:17:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:17:40 2026 2026-02-20T21:17:51.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:17:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ba958b175d 2026-02-20T21:17:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:17:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:17:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:17:51.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:17:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n snap9-clean ']' 2026-02-20T21:17:51.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap9-clean 2026-02-20T21:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9-clean 2026-02-20T21:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:17:51.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:17:51.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:17:51.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:17:51.955 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:51.953+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:51.957 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:51.954+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:51.957 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:51.955+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:51.957 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:51.955+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:51.957 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:51.955+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:51.958 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:51.955+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:52.043 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.041+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:52.043 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.041+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:52.044 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.041+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:52.075 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.073+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:52.076 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.073+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:17:52.076 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.074+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:52.077 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.074+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:52.077 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.075+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:52.112 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.109+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:17:52.112 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.109+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:52.112 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.110+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:17:52.113 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.110+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:52.143 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.140+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:17:52.144 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:52.141+0000 7f4129cc6640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:17:52.144 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:52.142+0000 7f412ecd0640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:17:52.146 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.142+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:52.146 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.142+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:52.146 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.143+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:52.146 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.143+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:52.183 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.180+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:17:52.184 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.180+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:52.184 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.180+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:52.184 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.180+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:17:52.184 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.181+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:17:52.249 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.246+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:17:52.249 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.246+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:17:52.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.247+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:17:52.286 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:52.283+0000 7f412dcce640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:17:52.286 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:52.283+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:17:52.286 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:52.283+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:17:52.287 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:52.284+0000 7f412c4cb640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:17:52.353 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:52.350+0000 7f412bcca640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:17:52.353 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:52.350+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:17:52.354 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:52.350+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:17:52.354 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:52.351+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:17:52.354 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:52.351+0000 7f412c4cb640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:17:52.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:17:52.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:17:52.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:17:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap10 2026-02-20T21:17:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10 2026-02-20T21:17:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:17:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:17:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:17:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:17:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:17:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:17:53.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:17:53.129 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.127+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:53.130 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.128+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:17:53.132 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.129+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:53.132 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.129+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:53.133 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.129+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:53.133 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.130+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:53.249 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:53.246+0000 7f412e4cf640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:17:53.250 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:53.246+0000 7f412ecd0640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:17:53.250 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:53.247+0000 7f4128cc4640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:17:53.251 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.248+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:17:53.286 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.283+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:17:53.288 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.284+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:53.288 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.284+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:53.288 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.285+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:53.288 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.285+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:53.289 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.285+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:53.393 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.390+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:53.393 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.390+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:53.394 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.391+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:17:53.394 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.391+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:17:53.394 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.391+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:17:53.431 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.428+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:17:53.431 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.428+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:17:53.432 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.429+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:17:53.432 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.429+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:17:53.433 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:17:53.430+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:17:53.501 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:53.498+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:17:53.501 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:53.498+0000 7f412bcca640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:17:53.503 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:53.500+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:17:53.504 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:53.501+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:17:53.504 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:17:53.501+0000 7f412c4cb640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:17:53.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:17:53.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:17:53.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:17:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ clean_snap_name=snap10-clean 2026-02-20T21:17:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror test snap10-clean 2026-02-20T21:17:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:17:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:53.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:53.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10-clean 2026-02-20T21:17:53.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create test@snap10-clean --debug-rbd=20 --debug-journaler=20 2026-02-20T21:17:54.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-20T21:17:54.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:54.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:54.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:54.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-20T21:17:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:17:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:17:54.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:54.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:17:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-20T21:17:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:17:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:17:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:17:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-20T21:17:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:17:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:17:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:17:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:17:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:17:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:55.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:17:55.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:55.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:17:55.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:55.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-20T21:17:55.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:17:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:17:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:17:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:17:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:17:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:17:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:17:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:17:55.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:17:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:17:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:17:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:17:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:17:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-20T21:17:55.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:55.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:17:55.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:55.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:55.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-20T21:17:55.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:17:55.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:17:55.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:55.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:56.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:56.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:56.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:56.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:56.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:56.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:17:56.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:17:56.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:56.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:56.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:56.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:17:56.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:17:56.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:17:56.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:56.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:17:56.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:56.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:56.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:17:56.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:56.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:56.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:17:56.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:17:56.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:17:56.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:17:56.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:56.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-20T21:17:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:17:56.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-20T21:17:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:17:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:17:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:17:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:17:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:17:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:56.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:17:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:17:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:17:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:17:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:17:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:17:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:17:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:56.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:56.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:17:56.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:17:56.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:17:56.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:17:56.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:56.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:56.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-20T21:17:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:17:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:17:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:17:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:17:56.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:17:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:17:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:17:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:17:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:17:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:17:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:17:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:17:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:17:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror test 2026-02-20T21:17:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:17:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:17:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:17:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:17:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:17:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:17:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:56.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:56.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:56.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:56.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:56.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:56.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:56.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:56.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:56.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:56.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:56.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:56.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:56.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:56.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:56.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:56.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:56.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:56.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:56.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:56.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:56.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:56.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:56.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:56.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:56.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:56.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:56.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:56.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:56.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:56.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:56.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:56.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:56.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:56.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:56.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:56.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:56.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:56.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:56.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:56.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:56.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:56.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:56.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:56.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:56.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:56.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:56.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:56.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:56.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:56.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:56.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:56.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=31, entry_tid=1' 2026-02-20T21:17:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:56.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:56.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:56.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:56.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:56.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:56.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=31, entry_tid=1' -a 'object_number=409, tag_tid=31, entry_tid=1' = 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=30, entry_tid=1' '!=' '' 2026-02-20T21:17:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:17:56.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=30, entry_tid=1' = '' 2026-02-20T21:17:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=31, entry_tid=1' 2026-02-20T21:17:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:56.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:56.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=31 2026-02-20T21:17:56.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:56.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:17:56.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:56.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=30 2026-02-20T21:17:56.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=31, entry_tid=1' 2026-02-20T21:17:56.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:56.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:56.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:17:56.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:56.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:17:56.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:17:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=1 2026-02-20T21:17:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 31 = 30 -a 1 -le 1 2026-02-20T21:17:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:17:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:17:56.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:56.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:56.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:56.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:56.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:56.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:56.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:56.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:56.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:56.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:56.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:56.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:56.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:56.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:56.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:56.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:56.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:56.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:56.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:56.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:56.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:56.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:56.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:56.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:56.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:56.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:56.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:56.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:56.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:56.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:56.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:56.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:56.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:56.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:56.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:56.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:56.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:56.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:56.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:56.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:56.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:56.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:56.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:56.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:56.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:56.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:56.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:56.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:56.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=31, entry_tid=1' 2026-02-20T21:17:56.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:56.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:56.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:56.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:57.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:57.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:57.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=31, entry_tid=1' -a 'object_number=409, tag_tid=31, entry_tid=1' = 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:57.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=30, entry_tid=1' '!=' 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:57.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:57.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:17:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:57.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:57.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:57.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:57.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:57.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:57.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:57.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:57.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:57.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:57.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:57.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:57.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:57.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:57.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:57.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:57.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:57.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:57.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:57.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:57.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:57.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:57.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:57.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:57.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:57.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:57.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:57.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:57.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:57.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:57.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:57.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:57.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=31, entry_tid=1' 2026-02-20T21:17:57.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:57.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:57.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:57.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:57.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:57.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:57.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:57.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:57.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:57.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:57.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:57.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:57.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:57.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:57.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=31, entry_tid=1' -a 'object_number=409, tag_tid=31, entry_tid=1' = 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=30, entry_tid=1' '!=' 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:17:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:17:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:17:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:17:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:17:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:17:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:17:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:17:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:17:58.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:17:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:17:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:17:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:17:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:17:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:17:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:17:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:17:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:17:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:58.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:17:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:17:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:17:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:17:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:58.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:17:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:17:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:17:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:17:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:17:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:17:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:58.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:58.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:17:58.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:17:58.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:17:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:58.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:17:58.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:17:58.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:58.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:58.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:17:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:17:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:17:58.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:17:58.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:17:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:17:58.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:17:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:17:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:17:58.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:17:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:17:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:17:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:17:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:17:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:17:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:17:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:17:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:17:58.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:17:58.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:17:58.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:17:58.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:58.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:17:58.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:58.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:17:58.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:58.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:17:58.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:17:58.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:17:58.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:17:58.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:17:58.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:17:58.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:17:58.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:17:58.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:17:58.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:17:58.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:17:58.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:17:58.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:17:58.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:17:58.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:17:58.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:17:58.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:58.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:58.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:17:58.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:17:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:17:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:17:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:17:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:17:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:58.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:58.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:58.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:58.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:58.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:58.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:58.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:58.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=31, entry_tid=1' 2026-02-20T21:17:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:17:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:17:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:17:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:17:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:17:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:17:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:17:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:17:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:17:58.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:58.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:17:58.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:58.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:17:58.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:17:58.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:17:58.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:58.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:17:58.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:17:58.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:58.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=31, entry_tid=1' -a 'object_number=409, tag_tid=31, entry_tid=1' = 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:58.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=30, entry_tid=1' '!=' 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:17:58.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:17:58.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:18:00.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:18:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:18:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:18:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:18:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:18:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:18:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:18:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:18:00.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:18:00.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:18:00.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:00.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:18:00.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:18:00.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:18:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:18:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:18:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:18:00.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:18:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:18:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:18:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:18:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:18:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:18:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:18:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:18:00.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:00.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:18:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:18:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:18:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:18:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:18:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:18:00.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:18:00.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:18:00.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:18:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:18:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:18:00.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:18:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:18:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:18:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:18:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:18:00.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:18:00.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:18:00.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:18:00.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:18:00.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:18:00.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:18:00.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:18:00.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:18:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:18:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:18:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:18:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:18:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:18:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:18:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:18:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:18:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:18:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:18:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:18:00.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:18:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:18:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:18:00.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:18:00.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:18:00.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:18:00.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:18:00.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:18:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:18:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:18:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:18:00.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:00.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:18:00.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:18:00.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:00.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:18:00.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:18:00.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:18:00.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:18:00.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:18:00.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:18:00.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:18:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:18:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:18:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:18:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:18:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:18:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:18:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:18:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:18:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:18:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:18:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:18:00.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:18:00.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:18:00.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:18:00.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:18:00.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:18:00.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:18:00.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:18:00.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:18:00.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:18:00.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:18:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:18:00.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:18:00.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:18:00.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:18:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:18:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:18:00.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:18:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:18:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:18:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:18:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:18:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:18:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:18:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:18:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:18:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:18:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:18:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:18:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:18:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:18:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:18:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:18:00.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:18:00.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:18:00.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:18:00.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=31, entry_tid=1' 2026-02-20T21:18:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:18:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:18:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:18:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:18:00.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:18:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:18:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:18:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:18:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:18:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:18:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:18:00.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:18:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:18:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:18:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:18:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:18:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:18:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:18:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:18:00.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=30, entry_tid=1], [object_number=408, tag_tid=30, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:18:00.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:18:00.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:18:00.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=31, entry_tid=1' -a 'object_number=409, tag_tid=31, entry_tid=1' = 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:18:00.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=409, tag_tid=30, entry_tid=1' '!=' 'object_number=409, tag_tid=30, entry_tid=1' 2026-02-20T21:18:00.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:18:00.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:18:02.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror test 2026-02-20T21:18:02.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:18:02.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:02.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:02.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:18:02.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n test ']' 2026-02-20T21:18:02.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-20T21:18:02.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-20T21:18:02.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:18:02.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:02.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:18:02.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:18:02.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:18:02.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:18:02.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:18:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:18:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:18:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-20T21:18:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:18:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:18:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:18:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:18:02.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:18:02.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:18:02.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:18:02.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:18:02.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:02.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:18:02.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:18:02.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:18:02.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:18:02.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:18:02.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:18:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:18:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:18:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:18:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:18:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:18:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:18:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:18:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:18:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:18:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:18:02.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:18:02.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:18:02.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:18:02.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:18:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:18:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-20T21:18:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:18:02.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:18:02.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:18:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test 2026-02-20T21:18:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:18:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:18:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:18:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:18:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:18:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:18:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:18:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:18:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:18:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:18:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:18:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:18:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:18:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:18:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:18:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:18:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:18:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:18:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:18:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:18:02.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:18:02.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:18:02.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:18:02.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:18:02.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:18:02.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:18:02.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:18:02.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:18:02.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:18:02.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:18:02.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:18:02.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:18:02.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test 2026-02-20T21:18:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-20T21:18:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:18:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:18:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:18:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/test 2026-02-20T21:18:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:18:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:18:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:18:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:18:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:18:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:18:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:18:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:18:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:18:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:18:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:18:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:18:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:18:02.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:18:02.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:18:02.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:18:02.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:18:02.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:18:02.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:18:02.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:18:02.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:18:02.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:18:02.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:18:02.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:18:02.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:18:02.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:18:02.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:18:02.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:18:02.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:18:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:18:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:18:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:18:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/test 2026-02-20T21:18:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-20T21:18:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:18:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:18:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:18:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-20T21:18:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:18:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:18:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:18:02.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:18:02.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:18:02.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:18:02.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:18:02.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:18:02.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:18:02.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:18:02.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:18:02.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:18:02.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:18:02.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:18:02.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:18:02.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:18:02.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=409, tag_tid=31, entry_tid=1' 2026-02-20T21:18:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-20T21:18:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:18:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:18:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:18:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-20T21:18:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:18:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:18:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:18:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:18:02.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-20T21:18:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-test.status 2026-02-20T21:18:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:18:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:18:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-20T21:18:02.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:18:02.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 102 2026-02-20T21:18:02.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 102 2026-02-20T21:18:02.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:18:02.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:18:02.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=409, tag_tid=31, entry_tid=1], [object_number=408, tag_tid=31, entry_tid=0], [object_number=411, tag_tid=29, entry_tid=41315], [object_number=410, tag_tid=29, entry_tid=41314]]], state=connected] 2026-02-20T21:18:02.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.status 2026-02-20T21:18:02.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=409, tag_tid=31, entry_tid=1' 2026-02-20T21:18:02.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=409, tag_tid=31, entry_tid=1' -a 'object_number=409, tag_tid=31, entry_tid=1' = 'object_number=409, tag_tid=31, entry_tid=1' 2026-02-20T21:18:02.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:02.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror test snap10-clean 2026-02-20T21:18:02.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:18:02.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:02.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:02.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10-clean 2026-02-20T21:18:02.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:18:02.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:18:02.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:18:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/test@snap10-clean 2026-02-20T21:18:03.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'test': 2026-02-20T21:18:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:18:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:18:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 20 2026-02-20T21:18:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ba958b175d 2026-02-20T21:18:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ba958b175d 2026-02-20T21:18:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:18:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:18:03.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:06:40 2026 2026-02-20T21:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:17:51 2026 2026-02-20T21:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:17:40 2026 2026-02-20T21:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ba958b175d 2026-02-20T21:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: ba923525-490d-4a76-8e5f-b742509a94da 2026-02-20T21:18:03.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:18:03.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:03.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 10 2026-02-20T21:18:03.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:03.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap1 2026-02-20T21:18:03.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap1 2026-02-20T21:18:03.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:03.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:03.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1 2026-02-20T21:18:03.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:03.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:03.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:03.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:03.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:04.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:04.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:04.168 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.166+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:04.170 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.167+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:04.170 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.168+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:04.170 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.168+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:04.171 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.168+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:04.171 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.168+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:04.322 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.319+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:04.323 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.320+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:04.324 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.320+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:04.324 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.320+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:04.324 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.321+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:04.419 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.416+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:04.420 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.416+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:04.420 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:04.417+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:04.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:04.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap1-clean 2026-02-20T21:18:04.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap1-clean 2026-02-20T21:18:04.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:04.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:04.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap1-clean 2026-02-20T21:18:04.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:04.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:04.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:04.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:04.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap1-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:05.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap1-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:05.234 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:05.231+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:05.235 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:05.232+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:05.235 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:05.232+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:05.236 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:05.232+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:05.236 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:05.233+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:05.236 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:05.233+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:05.395 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:05.392+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:05.396 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:05.393+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:05.397 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:05.394+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:05.397 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:05.394+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:05.397 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:05.394+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:05.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:05.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap2 2026-02-20T21:18:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap2 2026-02-20T21:18:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2 2026-02-20T21:18:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:05.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:06.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:06.328 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.325+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:06.329 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.326+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:06.329 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.326+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.330 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.327+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.331 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.328+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.420+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:06.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.420+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:06.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.421+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:06.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.421+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:06.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.421+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.494 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.492+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:06.497 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.494+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:06.497 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.495+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:06.498 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.495+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:06.498 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.495+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:06.500 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.497+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:06.500 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.497+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:06.525 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.522+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.525 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.522+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.525 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.522+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.525 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.522+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.594 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.592+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:06.595 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.593+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.596 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.593+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.596 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.593+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.597 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.594+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:06.597 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.594+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:06.597 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.594+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:06.630 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.627+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:06.631 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.628+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:06.632 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.629+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:06.632 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.629+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.632 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.629+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.632 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:06.629+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:06.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:06.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap2-clean 2026-02-20T21:18:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap2-clean 2026-02-20T21:18:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:07.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap2-clean 2026-02-20T21:18:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:07.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap2-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:07.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:07.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap2-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:07.492 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.489+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:07.492 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.489+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:07.492 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.489+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:07.492 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.489+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:07.494 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.491+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:07.494 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.491+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:07.566 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.564+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:07.567 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.564+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:07.567 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.564+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:07.568 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.565+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:07.637 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.634+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:07.638 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.635+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:07.638 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.635+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:07.638 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.635+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:07.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.635+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:07.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.733+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:07.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.733+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:07.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.733+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:07.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.733+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:07.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.733+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:07.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.733+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:07.769 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.766+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:07.769 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.766+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:07.769 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.766+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:07.770 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.767+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:07.770 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.767+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:07.771 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:07.768+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:07.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:08.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:08.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:08.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:08.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap3 2026-02-20T21:18:08.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap3 2026-02-20T21:18:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3 2026-02-20T21:18:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:08.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:08.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:08.611 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.608+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.613 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.610+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:08.614 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.610+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:08.614 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.610+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:08.614 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.610+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:08.614 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.611+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:08.776 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.773+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:08.777 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.774+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.778 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.774+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.778 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.775+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.778 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.775+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.779 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.775+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.779 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.775+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.805 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.802+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:08.806 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.803+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:08.806 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.803+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:08.807 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.804+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:08.807 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.804+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:08.874 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.871+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:08.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.872+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.872+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.872+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.872+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.872+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.877 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.873+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:08.911 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.908+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:08.911 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.908+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:08.911 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.909+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:08.911 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.909+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:08.911 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.909+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:08.982 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:08.979+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:09.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap3-clean 2026-02-20T21:18:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap3-clean 2026-02-20T21:18:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap3-clean 2026-02-20T21:18:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap3-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:09.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:09.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap3-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:09.781 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.778+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:09.782 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.779+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:09.782 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.779+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:09.782 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.779+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:09.784 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.781+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:09.785 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.782+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:09.858 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.855+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:09.859 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.856+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:09.859 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.856+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:09.859 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.856+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:09.860 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.856+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:09.926 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.924+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:09.927 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.924+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:09.929 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.926+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:09.929 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.926+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:09.929 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.926+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:09.930 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.926+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:09.962 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.959+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:09.963 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.960+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:09.965 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:09.962+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:10.032 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.029+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:10.033 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.030+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:10.033 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.031+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:10.035 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.031+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:10.035 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.031+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:10.035 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.032+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:10.035 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.032+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:10.073 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.070+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:10.074 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.071+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:10.074 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.071+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:10.074 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.071+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:10.075 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.072+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:10.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:10.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap4 2026-02-20T21:18:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap4 2026-02-20T21:18:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4 2026-02-20T21:18:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:10.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:10.970 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.967+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:10.971 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.967+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:10.971 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.967+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:10.971 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.967+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:10.972 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.970+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:10.973 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:10.970+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:11.058 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.055+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:11.059 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.056+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:11.128 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.126+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:11.130 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.127+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.130 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.127+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.130 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.127+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.131 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.127+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.131 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.127+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.167 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.164+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.167 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.164+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.167 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.164+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.167 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.164+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.167 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.164+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.234 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.231+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:11.236 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.232+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.236 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.232+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.236 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.233+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.236 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.233+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.236 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.233+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.236 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.233+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:11.270 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.268+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:11.270 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.268+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:11.271 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.268+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:11.271 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.268+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:11.272 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.269+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:11.272 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.269+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:11.339 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.337+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:11.340 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.337+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:11.340 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:11.337+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:11.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:11.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:11.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:11.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap4-clean 2026-02-20T21:18:11.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap4-clean 2026-02-20T21:18:11.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:11.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:11.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap4-clean 2026-02-20T21:18:11.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:11.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:11.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:11.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:11.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap4-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:12.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:12.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap4-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:12.154 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.151+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.154 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.151+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.155 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.151+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.155 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.151+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.155 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.151+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.156 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.154+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:12.228 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.225+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:12.228 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.225+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:12.262 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.260+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:12.263 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.260+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:12.263 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:12.260+0000 7f4128cc4640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:18:12.265 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.262+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.265 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.262+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.296 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.293+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:12.296 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.293+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:12.296 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.293+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:12.328 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.325+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:12.329 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.326+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.329 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.326+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.329 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.326+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.330 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.327+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.364 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.361+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:12.364 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.361+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:12.364 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.361+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:12.365 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.362+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.365 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.362+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.431 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.428+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:12.431 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.428+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:12.431 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.428+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:12.432 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.428+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:12.432 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.429+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:12.432 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.429+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:12.467 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.464+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:12.468 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.465+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:12.468 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.465+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:12.468 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.465+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:12.469 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:12.466+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:12.539 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:12.536+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:12.540 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:12.537+0000 7f412c4cb640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:18:12.541 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:12.537+0000 7f412fcd2640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:18:12.541 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:12.537+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:18:12.541 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:12.538+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:18:12.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:12.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:12.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap5 2026-02-20T21:18:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap5 2026-02-20T21:18:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5 2026-02-20T21:18:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:12.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:13.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:13.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:13.351 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.348+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:13.352 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.350+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.353 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.350+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.353 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.350+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.353 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.350+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.354 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.352+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.435 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.432+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:13.435 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.432+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:13.436 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.432+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:13.436 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.433+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:13.436 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.433+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:13.473 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:13.470+0000 7f4128cc4640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:18:13.473 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.470+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:13.474 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:13.470+0000 7f412ecd0640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:13.474 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:13.470+0000 7f4129cc6640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:13.474 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.471+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.475 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.471+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.475 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.471+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.475 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.471+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.538 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:13.536+0000 7f412cccc640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:18:13.539 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:13.536+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:13.539 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:13.537+0000 7f412dcce640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:13.540 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.537+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.541 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.537+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.541 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.537+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.541 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.538+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.542 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.540+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:13.573 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.570+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:13.573 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.570+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:13.573 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.570+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:13.574 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.570+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:13.574 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.571+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:13.638 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.635+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:13.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.636+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:13.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.636+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:13.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.636+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:13.640 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.637+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.640 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.637+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.673 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.671+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:13.674 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.671+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:13.675 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.672+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.675 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.672+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.675 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.672+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.675 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:13.672+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:13.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:13.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:13.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap5-clean 2026-02-20T21:18:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap5-clean 2026-02-20T21:18:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:14.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap5-clean 2026-02-20T21:18:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:14.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap5-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:14.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap5-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:14.565 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.562+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:14.568 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.565+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:14.568 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.565+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:14.568 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.565+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:14.569 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.566+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:14.569 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.566+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:14.644 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.641+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:14.644 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.641+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:14.645 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.641+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:14.653 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.650+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:14.654 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.651+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:14.727 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.724+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:14.728 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.725+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.728 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.725+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.728 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.725+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.729 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.727+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.760 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.758+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.760 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.758+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.761 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.758+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.761 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.758+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.762 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.759+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:14.799 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.796+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:14.827 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.824+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:14.828 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.825+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.828 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.825+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.828 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.825+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.828 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.825+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.828 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.825+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.863 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.860+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:14.864 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.861+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.864 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.861+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.865 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.861+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.865 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.862+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.865 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:14.862+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:14.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap6 2026-02-20T21:18:15.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap6 2026-02-20T21:18:15.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:15.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:15.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6 2026-02-20T21:18:15.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:15.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:15.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:15.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:15.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:15.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:15.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:15.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.639+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.639+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.639+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.639+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.639+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.639+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.790 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.787+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:15.791 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.788+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:15.791 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.788+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:15.791 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.788+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:15.791 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.788+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:15.791 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.788+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:15.862 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.858+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.862 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.858+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.862 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.858+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.862 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.858+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.862 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.858+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.862 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.859+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.862 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.859+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:15.882 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.880+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:15.883 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.880+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:15.883 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.880+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:15.884 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.880+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:15.884 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.881+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:15.940 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:15.937+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:15.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:16.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:16.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap6-clean 2026-02-20T21:18:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap6-clean 2026-02-20T21:18:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap6-clean 2026-02-20T21:18:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:16.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap6-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:16.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:16.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap6-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:16.686 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.683+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:16.687 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.684+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:16.687 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.684+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:16.688 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.684+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:16.688 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.684+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:16.688 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.684+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:16.813 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.810+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:16.813 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.810+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:16.815 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.812+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:16.816 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.812+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:16.816 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.813+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:16.816 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.813+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:16.849 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.845+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:16.849 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.846+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:16.849 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.846+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:16.849 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.846+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:16.891 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.887+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:16.891 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.887+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:16.891 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.887+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:16.891 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.887+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:16.891 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:16.887+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:16.926 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:16.923+0000 7f412fcd2640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:18:16.927 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:16.924+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:16.927 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:16.924+0000 7f41304d3640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:17.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:17.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:17.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap7 2026-02-20T21:18:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap7 2026-02-20T21:18:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7 2026-02-20T21:18:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:17.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:17.832 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:17.829+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:17.832 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:17.829+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:17.832 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:17.829+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:17.833 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:17.829+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:17.833 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:17.830+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:17.835 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:17.832+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:17.931 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:17.928+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:17.931 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:17.928+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:17.931 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:17.928+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:17.968 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:17.965+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:17.968 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:17.965+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:18.064 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:18.061+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:18.064 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:18.061+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:18.065 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:18.061+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:18.066 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:18.062+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:18.066 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:18.062+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:18.100 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:18.097+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:18.101 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:18.098+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:18.101 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:18.098+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:18.101 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:18.098+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:18.101 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:18.098+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:18.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:18.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:18.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap7-clean 2026-02-20T21:18:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap7-clean 2026-02-20T21:18:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap7-clean 2026-02-20T21:18:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:18.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:18.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap7-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:19.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap7-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:19.167 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.165+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:19.168 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.166+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:19.169 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.166+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:19.169 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.166+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:19.170 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.167+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:19.170 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.167+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:19.275 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.272+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:19.276 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.273+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:19.277 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.274+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:19.311 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.308+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:19.312 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.309+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:19.347 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.344+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:19.349 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.346+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:19.350 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.347+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:19.350 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.347+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:19.350 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.347+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:19.350 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.347+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:19.383 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.380+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:19.384 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.381+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:19.384 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.381+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:19.419 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.416+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:19.419 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.416+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:19.420 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.417+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:19.421 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.417+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:19.421 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.418+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:19.456 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.454+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:19.457 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.454+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:19.457 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.455+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:19.458 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.455+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:19.458 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:19.456+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:19.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:19.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap8 2026-02-20T21:18:19.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap8 2026-02-20T21:18:19.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:19.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:19.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8 2026-02-20T21:18:19.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:19.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:19.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:19.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:19.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:20.353 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.350+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:20.354 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.351+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:20.354 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.351+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:20.356 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.353+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.356 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.353+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.356 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.354+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.475 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.472+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:20.476 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:20.473+0000 7f41304d3640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:20.477 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:20.474+0000 7f412dcce640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:20.477 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.474+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.477 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.474+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.478 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.474+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.478 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.474+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.505 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.502+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:20.505 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.502+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:20.505 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.502+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:20.506 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.503+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:20.536 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.534+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:20.538 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.535+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.538 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.535+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.541 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.538+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:20.541 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.538+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:20.541 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.538+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 5 2026-02-20T21:18:20.574 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.571+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:20.574 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.571+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:20.610 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.607+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:20.611 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.608+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:20.611 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.608+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:20.611 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.608+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:20.612 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.609+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:20.644 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.642+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:20.645 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.642+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:20.645 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.642+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:20.645 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.642+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:20.646 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.643+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:20.646 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.643+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.677+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:20.682 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.679+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.682 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.679+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.682 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.679+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.682 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:20.679+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:20.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:20.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap8-clean 2026-02-20T21:18:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap8-clean 2026-02-20T21:18:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap8-clean 2026-02-20T21:18:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:21.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:21.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap8-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:21.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:21.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap8-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:21.540 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.537+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:21.540 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.537+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:21.541 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.537+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:21.541 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.537+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:21.541 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.537+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:21.541 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.537+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:21.620 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.617+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:21.620 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.617+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:21.621 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.617+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:21.621 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.617+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:21.659 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.656+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:21.659 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.657+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:21.660 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.657+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:21.695 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.692+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:21.696 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.693+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:21.696 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.693+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:21.697 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.694+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:21.735 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.732+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:21.735 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.732+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:21.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.733+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:21.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.733+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:21.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.733+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:21.805 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.802+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:21.805 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.802+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:21.806 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.802+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:21.806 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.803+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:21.806 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.803+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:21.836 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.833+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:21.837 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.833+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:21.837 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.833+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:21.837 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.834+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:21.837 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.834+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:21.837 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.834+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:21.873 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.871+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:21.874 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.871+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:21.875 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.872+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:21.875 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:21.872+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:21.937 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:21.934+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:21.937 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:21.934+0000 7f412fcd2640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:21.938 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:21.935+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:18:21.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:22.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap9 2026-02-20T21:18:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap9 2026-02-20T21:18:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9 2026-02-20T21:18:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:22.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:22.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:22.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:22.746 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.743+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:22.746 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.743+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:22.747 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.744+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:22.748 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.745+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:22.748 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.745+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:22.748 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.745+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:22.824 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.821+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:22.862 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:22.859+0000 7f412cccc640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:18:22.862 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:22.859+0000 7f41304d3640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:18:22.862 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:22.859+0000 7f4129cc6640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:18:22.864 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.861+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:22.864 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.861+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:22.864 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.861+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:22.864 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.861+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:22.938 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.935+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:22.938 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.935+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:22.940 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.937+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:22.941 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.937+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:22.941 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.937+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:22.974 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.971+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:22.975 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.971+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:22.975 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:22.971+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:23.011 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:23.008+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:23.011 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:23.008+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:23.043 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:18:23.040+0000 7fc9c2010640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 2 2026-02-20T21:18:23.044 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:18:23.041+0000 7fc9be809640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 2 2026-02-20T21:18:23.044 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:18:23.041+0000 7fc9c3012640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2026-02-20T21:18:23.044 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:18:23.041+0000 7fc9bc004640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 2 2026-02-20T21:18:23.148 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:23.145+0000 7f412c4cb640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:18:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:23.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:23.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:23.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap9-clean 2026-02-20T21:18:23.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap9-clean 2026-02-20T21:18:23.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:23.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:23.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap9-clean 2026-02-20T21:18:23.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:23.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap9-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:23.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:23.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap9-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:23.936 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:23.933+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:23.937 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:23.934+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:23.937 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:23.934+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:23.938 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:23.934+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:23.938 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:23.934+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:23.938 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:23.935+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.022 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.019+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:24.051 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.048+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:24.053 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.050+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.055 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.051+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.055 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.051+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.055 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.051+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.126 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.123+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:24.127 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.124+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:24.127 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.124+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:24.128 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.125+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.128 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.125+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.159 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.156+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:24.160 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.157+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:24.198 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.195+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:24.198 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.195+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:24.198 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.195+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:24.199 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.196+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.200 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.197+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.200 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.197+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.235 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.232+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:24.235 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.233+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:24.236 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.234+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:24.237 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.235+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.238 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.235+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.238 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:24.235+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:24.305 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:24.302+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:18:24.306 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:24.302+0000 7f412ecd0640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:18:24.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:24.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:24.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap10 2026-02-20T21:18:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap10 2026-02-20T21:18:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10 2026-02-20T21:18:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:24.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:24.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:24.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:24.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10 /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:25.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:25.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10 /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:25.105 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.103+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:25.107 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.104+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:25.107 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.104+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:25.108 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.105+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:25.108 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.105+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:25.109 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.106+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:25.179 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.177+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:25.180 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.177+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:25.180 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.177+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:25.210 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.207+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:25.210 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.207+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:25.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.244+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:25.249 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.246+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:25.277 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.274+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:25.278 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.275+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:25.278 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.275+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:25.278 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.275+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:25.279 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.276+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:25.279 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.276+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:25.351 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.348+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:25.352 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.349+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:25.353 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.350+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:25.353 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.350+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:25.353 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.350+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:25.354 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.350+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:25.354 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.351+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:25.448 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.445+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:25.448 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:25.445+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:25.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:25.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:25.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap10-clean 2026-02-20T21:18:25.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror test snap10-clean 2026-02-20T21:18:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap10-clean 2026-02-20T21:18:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:18:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export test@snap10-clean /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export 2026-02-20T21:18:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:26.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export test@snap10-clean /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:26.253 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.250+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:26.254 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.251+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.254 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.251+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.255 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.252+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.256 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.253+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:26.256 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.253+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:26.350 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.346+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:26.350 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.346+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:26.350 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.346+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:26.350 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.346+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:26.386 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.383+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:26.387 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:26.384+0000 7f4128cc4640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:18:26.387 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.384+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:26.388 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:26.384+0000 7f412c4cb640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:26.388 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:18:26.384+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:18:26.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.420+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.421+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:26.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.421+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:26.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.421+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:26.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.421+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:26.425 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.422+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:26.456 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.453+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.457 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.453+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.458 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.454+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:26.458 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.455+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:26.458 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.455+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:18:26.524 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.522+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:18:26.525 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.522+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:26.525 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.522+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:26.525 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.522+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.525 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.523+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.526 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.524+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.558 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.555+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.558 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.555+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.558 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.555+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.559 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.555+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.559 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.555+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:18:26.633 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.630+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:26.634 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.630+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:26.634 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:18:26.630+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:18:26.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2026-02-20T21:18:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:26.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-test.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-test.export 2026-02-20T21:18:27.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:27.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 10 2026-02-20T21:18:27.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:27.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap1 2026-02-20T21:18:27.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap1 2026-02-20T21:18:27.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:27.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:27.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:27.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap1 2026-02-20T21:18:27.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1 2026-02-20T21:18:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap1-clean 2026-02-20T21:18:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap1-clean 2026-02-20T21:18:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap1-clean 2026-02-20T21:18:27.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap1-clean 2026-02-20T21:18:28.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:28.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:28.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap2 2026-02-20T21:18:28.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap2 2026-02-20T21:18:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap2 2026-02-20T21:18:28.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2 2026-02-20T21:18:29.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap2-clean 2026-02-20T21:18:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap2-clean 2026-02-20T21:18:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:29.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap2-clean 2026-02-20T21:18:29.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap2-clean 2026-02-20T21:18:30.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap3 2026-02-20T21:18:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap3 2026-02-20T21:18:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap3 2026-02-20T21:18:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3 2026-02-20T21:18:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:31.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap3-clean 2026-02-20T21:18:31.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap3-clean 2026-02-20T21:18:31.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:31.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:31.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:31.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap3-clean 2026-02-20T21:18:31.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap3-clean 2026-02-20T21:18:32.269 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.34646949058305365) > weight (0.25) 2026-02-20T21:18:32.269 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-20T21:18:32.269 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-20T21:18:32.269 INFO:tasks.rbd_mirror_thrash:waiting for 3.129354721384179 secs before reviving daemons 2026-02-20T21:18:32.270 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:18:32.268+0000 7f0b2bf75640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 76088) UID: 1000 2026-02-20T21:18:32.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap4 2026-02-20T21:18:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap4 2026-02-20T21:18:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap4 2026-02-20T21:18:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4 2026-02-20T21:18:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap4-clean 2026-02-20T21:18:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap4-clean 2026-02-20T21:18:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap4-clean 2026-02-20T21:18:33.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap4-clean 2026-02-20T21:18:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap5 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap5 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap5 2026-02-20T21:18:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5 2026-02-20T21:18:35.400 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-20T21:18:35.400 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:18:35.400 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-20T21:18:35.400 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-20T21:18:35.401 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-20T21:18:35.403 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-20T21:18:35.403 INFO:tasks.rbd_mirror_thrash:waiting for 106.9792762497656 secs before thrashing 2026-02-20T21:18:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:35.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap5-clean 2026-02-20T21:18:35.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap5-clean 2026-02-20T21:18:35.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:35.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:35.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:35.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap5-clean 2026-02-20T21:18:35.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap5-clean 2026-02-20T21:18:36.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap6 2026-02-20T21:18:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap6 2026-02-20T21:18:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap6 2026-02-20T21:18:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6 2026-02-20T21:18:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap6-clean 2026-02-20T21:18:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap6-clean 2026-02-20T21:18:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap6-clean 2026-02-20T21:18:37.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap6-clean 2026-02-20T21:18:38.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap7 2026-02-20T21:18:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap7 2026-02-20T21:18:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap7 2026-02-20T21:18:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7 2026-02-20T21:18:39.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap7-clean 2026-02-20T21:18:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap7-clean 2026-02-20T21:18:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap7-clean 2026-02-20T21:18:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap7-clean 2026-02-20T21:18:40.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap8 2026-02-20T21:18:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap8 2026-02-20T21:18:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap8 2026-02-20T21:18:40.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8 2026-02-20T21:18:41.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap8-clean 2026-02-20T21:18:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap8-clean 2026-02-20T21:18:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:41.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:41.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:41.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap8-clean 2026-02-20T21:18:41.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap8-clean 2026-02-20T21:18:42.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:43.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:43.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap9 2026-02-20T21:18:43.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap9 2026-02-20T21:18:43.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:43.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:43.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:43.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap9 2026-02-20T21:18:43.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9 2026-02-20T21:18:43.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap9-clean 2026-02-20T21:18:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap9-clean 2026-02-20T21:18:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap9-clean 2026-02-20T21:18:44.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap9-clean 2026-02-20T21:18:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:18:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap10 2026-02-20T21:18:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap10 2026-02-20T21:18:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap10 2026-02-20T21:18:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10 2026-02-20T21:18:45.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap10-clean 2026-02-20T21:18:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror test snap10-clean 2026-02-20T21:18:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap10-clean 2026-02-20T21:18:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/test@snap10-clean 2026-02-20T21:18:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:18:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror test 2026-02-20T21:18:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:18:46.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:18:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror test 2026-02-20T21:18:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:46.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-20T21:18:47.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-20T21:18:47.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:18:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-20T21:18:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:18:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:47.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:18:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:18:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:18:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:18:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:18:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:18:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:18:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:18:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:18:47.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:18:47.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:18:47.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:18:47.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:18:47.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:18:47.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror test deleted '' 2026-02-20T21:18:47.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:18:47.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:47.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=test 2026-02-20T21:18:47.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:18:47.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:18:47.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:18:47.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:18:47.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror test 2026-02-20T21:18:47.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:18:47.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:18:47.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=test 2026-02-20T21:18:47.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-20T21:18:47.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:18:47.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-20T21:18:47.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:18:47.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:18:47.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:18:47.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:18:47.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: create many images' 2026-02-20T21:18:47.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.IQNOxgiLiE/rbd-mirror.test.log 2026-02-20T21:18:47.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-20T21:18:47.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-20 21:18:47 TEST: create many images 2026-02-20T21:18:47.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-20 21:18:47 TEST: create many images 2026-02-20T21:18:47.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ snap_name=snap 2026-02-20T21:18:47.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 50 2026-02-20T21:18:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:18:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_1 2026-02-20T21:18:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_1 journal 128M 2026-02-20T21:18:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:47.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:18:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:18:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:18:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_1 128M 2026-02-20T21:18:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:18:47.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:18:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:18:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:18:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:18:47.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_1 2026-02-20T21:18:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:18:47.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:18:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:18:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:18:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:18:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_1 2026-02-20T21:18:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:18:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:18:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:18:47.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:18:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-20T21:18:47.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:18:47.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:18:48.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 96089088 bytes 2026-02-20T21:18:49.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 139283456 bytes 2026-02-20T21:18:51.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 196295680 bytes 2026-02-20T21:18:51.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 241063936 bytes 2026-02-20T21:18:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 283068416 bytes 2026-02-20T21:18:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 327524352 bytes 2026-02-20T21:18:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 348739584 bytes 2026-02-20T21:18:53.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 387440640 bytes 2026-02-20T21:18:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 450069504 bytes 2026-02-20T21:18:54.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 453107712 bytes 2026-02-20T21:18:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 484060160 bytes 2026-02-20T21:18:54.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 497769472 bytes 2026-02-20T21:18:55.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 515765248 bytes 2026-02-20T21:18:55.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 516387840 bytes 2026-02-20T21:18:55.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 520483840 bytes 2026-02-20T21:18:55.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 585962496 bytes 2026-02-20T21:18:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 590520320 bytes 2026-02-20T21:18:56.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 614563840 bytes 2026-02-20T21:18:56.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 665942016 bytes 2026-02-20T21:18:58.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 734497792 bytes 2026-02-20T21:18:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 816661504 bytes 2026-02-20T21:18:59.802 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6798127075451506) > weight (0.25) 2026-02-20T21:18:59.802 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.4108112651316468) > weight (0.25) 2026-02-20T21:18:59.802 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.989839743290239) > weight (0.25) 2026-02-20T21:18:59.803 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.39321260460782703) > weight (0.25) 2026-02-20T21:18:59.803 INFO:tasks.rbd_mirror_thrash:waiting for 117.06939874039574 secs before thrashing 2026-02-20T21:18:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2200 IOs, 847427584 bytes 2026-02-20T21:19:00.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2300 IOs, 850156544 bytes 2026-02-20T21:19:00.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2400 IOs, 899239936 bytes 2026-02-20T21:19:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2500 IOs, 910698496 bytes 2026-02-20T21:19:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:19:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:19:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:19:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:19:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:19:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_2 2026-02-20T21:19:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_2 journal 128M 2026-02-20T21:19:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:01.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_2 128M 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:01.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_2 2026-02-20T21:19:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:19:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:19:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:19:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:19:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:19:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_2 2026-02-20T21:19:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:19:01.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:19:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-20T21:19:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:19:01.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:19:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 18790400 bytes 2026-02-20T21:19:02.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 109071360 bytes 2026-02-20T21:19:03.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 136198144 bytes 2026-02-20T21:19:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 207201280 bytes 2026-02-20T21:19:05.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 271938560 bytes 2026-02-20T21:19:06.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 296526848 bytes 2026-02-20T21:19:07.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 376933376 bytes 2026-02-20T21:19:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 418985984 bytes 2026-02-20T21:19:08.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 491364352 bytes 2026-02-20T21:19:09.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:19:09.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:19:09.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:19:09.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:19:09.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:19:09.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_3 2026-02-20T21:19:09.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_3 journal 128M 2026-02-20T21:19:09.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:19:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:19:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:19:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_3 128M 2026-02-20T21:19:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:09.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:19:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:19:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:19:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:19:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:09.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_3 2026-02-20T21:19:09.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:19:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:19:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:19:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:19:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:19:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_3 2026-02-20T21:19:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:19:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:19:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-20T21:19:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:19:09.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:19:10.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 42693632 bytes 2026-02-20T21:19:10.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 86483968 bytes 2026-02-20T21:19:11.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 121996288 bytes 2026-02-20T21:19:12.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 202900480 bytes 2026-02-20T21:19:13.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 256391168 bytes 2026-02-20T21:19:14.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 279443456 bytes 2026-02-20T21:19:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 356724736 bytes 2026-02-20T21:19:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 377077760 bytes 2026-02-20T21:19:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 403482624 bytes 2026-02-20T21:19:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 447026176 bytes 2026-02-20T21:19:17.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 522715136 bytes 2026-02-20T21:19:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 595954688 bytes 2026-02-20T21:19:19.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 609801216 bytes 2026-02-20T21:19:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 641614848 bytes 2026-02-20T21:19:20.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 710887424 bytes 2026-02-20T21:19:21.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 743101440 bytes 2026-02-20T21:19:22.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 836158464 bytes 2026-02-20T21:19:22.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 885463040 bytes 2026-02-20T21:19:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 925438976 bytes 2026-02-20T21:19:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:19:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:19:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:19:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:19:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:19:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_4 2026-02-20T21:19:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_4 journal 128M 2026-02-20T21:19:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:19:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:19:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:19:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_4 128M 2026-02-20T21:19:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:23.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:19:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:19:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:19:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:19:23.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:23.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_4 2026-02-20T21:19:23.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:19:23.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:23.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:19:23.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:19:23.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:19:23.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:19:23.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_4 2026-02-20T21:19:23.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:23.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:23.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:19:23.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:19:23.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-20T21:19:23.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:19:23.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:19:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 54762496 bytes 2026-02-20T21:19:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 95461376 bytes 2026-02-20T21:19:25.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 168559616 bytes 2026-02-20T21:19:26.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 206809088 bytes 2026-02-20T21:19:27.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 240520192 bytes 2026-02-20T21:19:27.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 267593728 bytes 2026-02-20T21:19:27.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 268233728 bytes 2026-02-20T21:19:28.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 300472320 bytes 2026-02-20T21:19:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 367828992 bytes 2026-02-20T21:19:29.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 438987776 bytes 2026-02-20T21:19:30.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 457028608 bytes 2026-02-20T21:19:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 513304576 bytes 2026-02-20T21:19:31.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 565882880 bytes 2026-02-20T21:19:32.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 601808896 bytes 2026-02-20T21:19:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 661668864 bytes 2026-02-20T21:19:33.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 731210752 bytes 2026-02-20T21:19:34.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 771110912 bytes 2026-02-20T21:19:35.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 836818944 bytes 2026-02-20T21:19:35.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 876566528 bytes 2026-02-20T21:19:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 907621376 bytes 2026-02-20T21:19:36.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 934315008 bytes 2026-02-20T21:19:37.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2200 IOs, 963525632 bytes 2026-02-20T21:19:37.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_5 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_5 journal 128M 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_5 128M 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:19:37.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:19:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:19:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:19:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:37.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_5 2026-02-20T21:19:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:19:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:19:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:19:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:19:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:19:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_5 2026-02-20T21:19:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:19:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:19:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-20T21:19:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:19:37.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:19:38.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 56269824 bytes 2026-02-20T21:19:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 74606592 bytes 2026-02-20T21:19:39.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 108069888 bytes 2026-02-20T21:19:40.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 120957952 bytes 2026-02-20T21:19:40.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 170586112 bytes 2026-02-20T21:19:41.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 226349056 bytes 2026-02-20T21:19:41.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 255260672 bytes 2026-02-20T21:19:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 290657280 bytes 2026-02-20T21:19:43.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 351896576 bytes 2026-02-20T21:19:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 352492544 bytes 2026-02-20T21:19:43.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 383931392 bytes 2026-02-20T21:19:44.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 458670080 bytes 2026-02-20T21:19:45.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 518121472 bytes 2026-02-20T21:19:45.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 518748160 bytes 2026-02-20T21:19:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 576384000 bytes 2026-02-20T21:19:47.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 650990592 bytes 2026-02-20T21:19:47.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 682447872 bytes 2026-02-20T21:19:47.809 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:19:47.806+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:19:47.810 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:19:47.807+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:19:47.810 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:19:47.807+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:19:48.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 696639488 bytes 2026-02-20T21:19:48.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 720191488 bytes 2026-02-20T21:19:49.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 766166016 bytes 2026-02-20T21:19:50.221 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:19:50.218+0000 7fd67aee6640 -1 reset not still connected to 0x55daafea72b0 2026-02-20T21:19:50.222 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:19:50.218+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423450 2026-02-20T21:19:50.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:19:50.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:19:50.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:19:50.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:19:50.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:19:50.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_6 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_6 journal 128M 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_6 128M 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:19:50.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_6 2026-02-20T21:19:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:19:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:19:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:19:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:19:50.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:19:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_6 2026-02-20T21:19:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:19:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:19:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:19:50.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:19:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-20T21:19:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:19:50.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:19:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 32913408 bytes 2026-02-20T21:19:52.116 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:19:52.113+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:19:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 102011904 bytes 2026-02-20T21:19:52.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 151393280 bytes 2026-02-20T21:19:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 166467584 bytes 2026-02-20T21:19:53.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 208025600 bytes 2026-02-20T21:19:54.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 222026752 bytes 2026-02-20T21:19:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 228541440 bytes 2026-02-20T21:19:54.756 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:19:54.753+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:19:54.757 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:19:54.753+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:19:54.757 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:19:54.753+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:19:55.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 297661440 bytes 2026-02-20T21:19:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 314268672 bytes 2026-02-20T21:19:56.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 346680320 bytes 2026-02-20T21:19:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 376615936 bytes 2026-02-20T21:19:57.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 399154176 bytes 2026-02-20T21:19:57.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 405888000 bytes 2026-02-20T21:19:57.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 412650496 bytes 2026-02-20T21:19:58.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 459984896 bytes 2026-02-20T21:19:58.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 480174080 bytes 2026-02-20T21:19:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 518325248 bytes 2026-02-20T21:19:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 555809792 bytes 2026-02-20T21:20:01.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 601874432 bytes 2026-02-20T21:20:02.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 695285760 bytes 2026-02-20T21:20:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 712026112 bytes 2026-02-20T21:20:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2200 IOs, 774980608 bytes 2026-02-20T21:20:03.811 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:03.808+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc92b0 2026-02-20T21:20:03.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2300 IOs, 786259968 bytes 2026-02-20T21:20:04.599 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:04.596+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:20:04.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:20:04.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:20:04.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_7 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_7 journal 128M 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_7 128M 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:04.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_7 2026-02-20T21:20:04.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:20:04.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:04.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:20:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:20:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:20:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:20:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_7 2026-02-20T21:20:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:20:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:20:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-20T21:20:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:20:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:20:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 47322112 bytes 2026-02-20T21:20:06.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 110491648 bytes 2026-02-20T21:20:07.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 144497664 bytes 2026-02-20T21:20:08.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 227295232 bytes 2026-02-20T21:20:09.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 266083328 bytes 2026-02-20T21:20:09.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 292375552 bytes 2026-02-20T21:20:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 318046208 bytes 2026-02-20T21:20:09.787 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:20:09.785+0000 7f03b504e640 -1 bdev(0x55af97a4d000 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2026-02-20T21:20:09.788 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:20:09.786+0000 7f03b584f640 -1 bdev(0x55af97a4d000 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2026-02-20T21:20:09.789 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:20:09.787+0000 7f03b9056640 -1 bdev(0x55af97a4d000 /var/lib/ceph/osd/cluster2-0/block) aio_submit retries 5 2026-02-20T21:20:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 324189184 bytes 2026-02-20T21:20:10.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 387073024 bytes 2026-02-20T21:20:10.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 396176384 bytes 2026-02-20T21:20:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 412310528 bytes 2026-02-20T21:20:12.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 543986688 bytes 2026-02-20T21:20:13.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 590798848 bytes 2026-02-20T21:20:14.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 616899584 bytes 2026-02-20T21:20:14.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 643148800 bytes 2026-02-20T21:20:14.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 667882496 bytes 2026-02-20T21:20:15.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 690492416 bytes 2026-02-20T21:20:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 743806976 bytes 2026-02-20T21:20:16.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 755702784 bytes 2026-02-20T21:20:16.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:20:16.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:20:16.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:20:16.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:20:16.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_8 2026-02-20T21:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_8 journal 128M 2026-02-20T21:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:16.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_8 128M 2026-02-20T21:20:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:20:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:20:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:20:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:20:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:16.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_8 2026-02-20T21:20:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:20:16.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:20:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:20:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:20:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:20:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_8 2026-02-20T21:20:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:20:16.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:20:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-20T21:20:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:20:16.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:20:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 10218496 bytes 2026-02-20T21:20:17.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 10830848 bytes 2026-02-20T21:20:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 63544320 bytes 2026-02-20T21:20:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 86340608 bytes 2026-02-20T21:20:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 127623168 bytes 2026-02-20T21:20:19.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 163057664 bytes 2026-02-20T21:20:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 223265792 bytes 2026-02-20T21:20:22.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 290074624 bytes 2026-02-20T21:20:22.383 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-20T21:20:22.384 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-20T21:20:22.384 INFO:tasks.rbd_mirror_thrash:waiting for 8.971075009916794 secs before reviving daemons 2026-02-20T21:20:22.385 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:20:22.382+0000 7f9a863f7640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 65297) UID: 1000 2026-02-20T21:20:22.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 314755072 bytes 2026-02-20T21:20:22.474 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:20:22.471+0000 7f03c8435640 -1 reset not still connected to 0x55afb9b1a1a0 2026-02-20T21:20:23.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 394900480 bytes 2026-02-20T21:20:23.809 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:20:23.805+0000 7f03c8435640 -1 reset not still connected to 0x55afb9b1a1a0 2026-02-20T21:20:23.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 416798720 bytes 2026-02-20T21:20:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 417906688 bytes 2026-02-20T21:20:24.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 435234816 bytes 2026-02-20T21:20:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 492725248 bytes 2026-02-20T21:20:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 523735040 bytes 2026-02-20T21:20:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 552118272 bytes 2026-02-20T21:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_9 2026-02-20T21:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_9 journal 128M 2026-02-20T21:20:26.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_9 128M 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:26.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_9 2026-02-20T21:20:26.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:20:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:20:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:20:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:20:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:20:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_9 2026-02-20T21:20:26.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:20:26.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:20:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=9 2026-02-20T21:20:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:20:26.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:20:28.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 92423168 bytes 2026-02-20T21:20:28.194 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:28.191+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:20:28.194 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:28.191+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:20:28.194 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:28.191+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:20:28.194 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:28.191+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:20:28.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 133533696 bytes 2026-02-20T21:20:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 155101184 bytes 2026-02-20T21:20:31.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 220969984 bytes 2026-02-20T21:20:31.356 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-20T21:20:31.356 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:20:31.357 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-20T21:20:31.357 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T21:20:31.357 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T21:20:31.359 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T21:20:31.360 INFO:tasks.rbd_mirror_thrash:waiting for 111.32576334476047 secs before thrashing 2026-02-20T21:20:32.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 273622016 bytes 2026-02-20T21:20:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 305884160 bytes 2026-02-20T21:20:33.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 331899904 bytes 2026-02-20T21:20:33.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 372739072 bytes 2026-02-20T21:20:34.256 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:34.253+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:20:34.256 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:34.253+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:20:34.256 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:34.253+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:20:34.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 428617728 bytes 2026-02-20T21:20:35.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:20:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:20:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:20:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:20:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:20:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_10 2026-02-20T21:20:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_10 journal 128M 2026-02-20T21:20:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:35.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:20:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:20:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:20:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_10 128M 2026-02-20T21:20:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:35.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:20:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:20:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:20:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:20:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:35.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_10 2026-02-20T21:20:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:20:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:20:35.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:20:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:20:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:20:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_10 2026-02-20T21:20:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:20:35.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:20:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-20T21:20:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:20:35.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:20:35.994 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:35.991+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:20:35.995 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:35.991+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:20:35.995 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:35.992+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:20:35.995 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:35.992+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:20:36.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 51792896 bytes 2026-02-20T21:20:36.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 76884992 bytes 2026-02-20T21:20:37.095 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:37.092+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:20:37.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 91386880 bytes 2026-02-20T21:20:37.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 91988992 bytes 2026-02-20T21:20:38.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 152539136 bytes 2026-02-20T21:20:39.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 190222336 bytes 2026-02-20T21:20:39.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 228799488 bytes 2026-02-20T21:20:39.816 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:39.813+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:20:39.816 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:39.813+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:20:39.817 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:39.813+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:20:39.817 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:39.813+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:20:39.817 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:39.813+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:20:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 285512704 bytes 2026-02-20T21:20:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 327809024 bytes 2026-02-20T21:20:42.220 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:42.216+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb32b0 2026-02-20T21:20:42.220 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:42.217+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe4704e0 2026-02-20T21:20:42.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 380609536 bytes 2026-02-20T21:20:42.880 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:42.877+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:20:42.880 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:42.877+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:20:42.881 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:42.877+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:20:42.926 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:42.923+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:20:42.926 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:42.923+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:20:42.927 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:42.924+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:20:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 428794880 bytes 2026-02-20T21:20:43.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 443724800 bytes 2026-02-20T21:20:44.486 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:44.483+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31432b0 2026-02-20T21:20:44.486 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:44.483+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb20d0 2026-02-20T21:20:44.487 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:44.484+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb32b0 2026-02-20T21:20:44.487 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:44.484+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe4704e0 2026-02-20T21:20:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 526483456 bytes 2026-02-20T21:20:45.373 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:45.369+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:20:45.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 564755456 bytes 2026-02-20T21:20:45.505 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:45.502+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc92b0 2026-02-20T21:20:46.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 604624896 bytes 2026-02-20T21:20:46.864 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:20:46.861+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:20:46.868 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:46.865+0000 7fd67aee6640 -1 reset not still connected to 0x55daafea72b0 2026-02-20T21:20:46.869 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:46.865+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423450 2026-02-20T21:20:46.869 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:46.865+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423930 2026-02-20T21:20:47.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 648343552 bytes 2026-02-20T21:20:47.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 648933376 bytes 2026-02-20T21:20:47.616 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:47.613+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:20:47.616 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:47.613+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:20:47.618 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:47.614+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:20:47.618 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:47.614+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:20:47.618 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:47.614+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:20:47.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:20:47.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:20:47.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:20:47.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:20:47.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:20:47.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_11 2026-02-20T21:20:47.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_11 journal 128M 2026-02-20T21:20:47.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:47.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:47.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:47.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:47.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:20:47.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:47.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:20:47.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:20:47.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:47.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_11 128M 2026-02-20T21:20:47.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:47.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:47.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:47.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:47.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:20:47.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:47.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:20:47.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:20:47.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:20:47.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:47.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_11 2026-02-20T21:20:47.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:20:47.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:47.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:20:47.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:20:47.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:20:47.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:20:47.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_11 2026-02-20T21:20:47.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:47.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:47.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:20:47.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:20:47.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=9 2026-02-20T21:20:47.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:20:47.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_11 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:20:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 15267840 bytes 2026-02-20T21:20:48.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 55249920 bytes 2026-02-20T21:20:49.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 93053952 bytes 2026-02-20T21:20:49.762 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:49.759+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc92b0 2026-02-20T21:20:49.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 123679744 bytes 2026-02-20T21:20:49.985 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:49.981+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc92b0 2026-02-20T21:20:50.567 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:50.564+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:20:50.568 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:50.564+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:20:51.955 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:20:51.952+0000 7f413d8ae640 -1 reset not still connected to 0x55d271027450 2026-02-20T21:20:51.956 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:20:51.952+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb380 2026-02-20T21:20:51.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 177140736 bytes 2026-02-20T21:20:52.730 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:52.726+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc92b0 2026-02-20T21:20:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 234500096 bytes 2026-02-20T21:20:53.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 249332736 bytes 2026-02-20T21:20:53.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 297738240 bytes 2026-02-20T21:20:54.393 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:20:54.391+0000 7f03c8435640 -1 reset not still connected to 0x55af9bcffba0 2026-02-20T21:20:54.393 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:20:54.391+0000 7f03c8435640 -1 reset not still connected to 0x55afce0ee1a0 2026-02-20T21:20:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 336816128 bytes 2026-02-20T21:20:54.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 337436672 bytes 2026-02-20T21:20:55.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 392427520 bytes 2026-02-20T21:20:55.222 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:55.219+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:20:55.224 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:55.221+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:20:55.224 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:55.221+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:20:55.224 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:55.221+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:20:55.225 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:55.221+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:20:55.486 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:55.483+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423450 2026-02-20T21:20:55.487 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:55.483+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423930 2026-02-20T21:20:55.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 393034752 bytes 2026-02-20T21:20:55.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 393631744 bytes 2026-02-20T21:20:56.078 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:56.075+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423930 2026-02-20T21:20:56.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 472438784 bytes 2026-02-20T21:20:56.873 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9070619788421282) > weight (0.25) 2026-02-20T21:20:56.873 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.1 2026-02-20T21:20:56.873 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Sent signal 15 2026-02-20T21:20:56.873 INFO:tasks.rbd_mirror_thrash:waiting for 7.633471490884905 secs before reviving daemons 2026-02-20T21:20:56.874 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:20:56.872+0000 7f4705631640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 76591) UID: 1000 2026-02-20T21:20:56.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:20:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:20:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:20:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:20:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:20:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_12 2026-02-20T21:20:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_12 journal 128M 2026-02-20T21:20:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:56.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_12 128M 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:20:56.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:20:56.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:20:56.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:20:56.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_12 2026-02-20T21:20:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:20:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:20:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:20:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:20:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:20:56.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_12 2026-02-20T21:20:56.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:20:56.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:20:56.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:20:56.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:20:56.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-20T21:20:56.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:20:56.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_12 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:20:57.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 28439552 bytes 2026-02-20T21:20:57.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 50691072 bytes 2026-02-20T21:20:58.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 97923072 bytes 2026-02-20T21:20:58.432 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:58.428+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:20:58.432 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:58.428+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:20:58.432 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:58.429+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:20:58.433 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:20:58.429+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:20:58.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 115830784 bytes 2026-02-20T21:20:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 148973568 bytes 2026-02-20T21:20:59.574 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:59.570+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:20:59.574 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:59.570+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:20:59.574 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:59.571+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:20:59.576 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:20:59.573+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 195014656 bytes 2026-02-20T21:21:00.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 195655680 bytes 2026-02-20T21:21:01.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 236593152 bytes 2026-02-20T21:21:01.066 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:21:01.064+0000 7f03c8435640 -1 reset not still connected to 0x55af9f5496c0 2026-02-20T21:21:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 237179904 bytes 2026-02-20T21:21:01.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 254104576 bytes 2026-02-20T21:21:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 302324736 bytes 2026-02-20T21:21:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 325986304 bytes 2026-02-20T21:21:02.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 353926144 bytes 2026-02-20T21:21:03.047 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:03.043+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:21:03.163 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:03.160+0000 7fd67aee6640 -1 reset not still connected to 0x55daae439ad0 2026-02-20T21:21:03.164 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:03.160+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:21:03.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 383294464 bytes 2026-02-20T21:21:03.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 413327360 bytes 2026-02-20T21:21:04.507 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.1 2026-02-20T21:21:04.507 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:21:04.508 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.1 2026-02-20T21:21:04.508 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Restarting daemon 2026-02-20T21:21:04.508 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 2026-02-20T21:21:04.511 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Started 2026-02-20T21:21:04.511 INFO:tasks.rbd_mirror_thrash:waiting for 85.97176415461387 secs before thrashing 2026-02-20T21:21:04.546 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:04.543+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb20d0 2026-02-20T21:21:04.548 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:04.544+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb32b0 2026-02-20T21:21:04.548 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:04.544+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe4704e0 2026-02-20T21:21:05.018 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:05.015+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:21:05.019 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:05.015+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:21:05.019 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:05.015+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:21:05.019 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:05.015+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:21:05.019 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:05.015+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:05.019 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:05.015+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:05.020 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:05.015+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:05.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 482246656 bytes 2026-02-20T21:21:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 524280832 bytes 2026-02-20T21:21:06.496 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:06.493+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f8340 2026-02-20T21:21:06.496 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:06.493+0000 7fd67aee6640 -1 reset not still connected to 0x55daafea72b0 2026-02-20T21:21:06.496 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:06.493+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423450 2026-02-20T21:21:06.496 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:06.493+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423930 2026-02-20T21:21:06.567 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:06.564+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:06.567 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:06.564+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:06.568 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:06.565+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:06.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 589019136 bytes 2026-02-20T21:21:07.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 661842944 bytes 2026-02-20T21:21:08.022 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:08.019+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:21:08.553 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:08.550+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:21:08.554 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:08.551+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:21:08.554 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:08.551+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:21:08.554 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:08.551+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:21:08.554 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:08.551+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:21:08.555 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:08.552+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:21:08.555 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:08.552+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:21:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 743183360 bytes 2026-02-20T21:21:09.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 745313280 bytes 2026-02-20T21:21:09.809 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:09.806+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:21:09.809 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:09.806+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:21:09.809 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:09.806+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:09.810 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:09.807+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:09.810 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:09.807+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_13 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_13 journal 128M 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_13 128M 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:09.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_13 2026-02-20T21:21:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:21:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:21:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:21:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:21:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:21:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_13 2026-02-20T21:21:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:21:10.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:21:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=11 2026-02-20T21:21:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:21:10.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_13 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:21:10.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 21474304 bytes 2026-02-20T21:21:11.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 42771456 bytes 2026-02-20T21:21:11.699 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:11.696+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:21:11.700 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:11.697+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:21:11.700 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:11.697+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:21:11.700 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:11.697+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:11.700 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:11.697+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:11.701 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:11.698+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:11.940 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:11.937+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423450 2026-02-20T21:21:11.940 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:11.937+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423930 2026-02-20T21:21:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 103724032 bytes 2026-02-20T21:21:13.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 130921472 bytes 2026-02-20T21:21:13.934 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:13.931+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:21:13.935 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:13.931+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:13.935 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:13.932+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:13.941 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:13.938+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:13.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 165424128 bytes 2026-02-20T21:21:14.509 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:14.505+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb20d0 2026-02-20T21:21:14.510 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:14.506+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb32b0 2026-02-20T21:21:14.510 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:14.506+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe4704e0 2026-02-20T21:21:14.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 215799808 bytes 2026-02-20T21:21:15.127 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:15.124+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb32b0 2026-02-20T21:21:15.127 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:15.124+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe4704e0 2026-02-20T21:21:15.420 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:15.416+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:21:15.420 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:15.416+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:21:15.420 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:15.416+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:21:15.420 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:15.416+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:21:15.420 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:15.416+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:15.420 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:15.416+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:15.421 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:15.416+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 267150336 bytes 2026-02-20T21:21:16.304 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:16.300+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:21:16.304 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:16.300+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:21:16.304 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:16.300+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:21:16.304 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:16.300+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:21:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 361526272 bytes 2026-02-20T21:21:17.029 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:17.025+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:17.030 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:17.027+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:17.043 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:17.039+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:21:17.043 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:17.039+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:21:17.043 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:17.039+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:17.043 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:17.039+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:17.044 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:17.041+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:18.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 455049216 bytes 2026-02-20T21:21:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 487499776 bytes 2026-02-20T21:21:19.521 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:21:19.518+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb380 2026-02-20T21:21:19.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 539088896 bytes 2026-02-20T21:21:20.029 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:20.026+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:20.029 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:20.026+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:20.030 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:20.027+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 590884864 bytes 2026-02-20T21:21:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:21:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:21:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:21:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:21:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:21:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_14 2026-02-20T21:21:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_14 journal 128M 2026-02-20T21:21:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_14 128M 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_14 2026-02-20T21:21:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:21:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:21:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:21:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:21:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:21:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_14 2026-02-20T21:21:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:21:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:21:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=11 2026-02-20T21:21:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:21:21.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_14 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:21:21.510 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:21.507+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:21:21.511 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:21.507+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:21:22.368 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:22.364+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f8340 2026-02-20T21:21:22.368 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:22.364+0000 7fd67aee6640 -1 reset not still connected to 0x55daafea72b0 2026-02-20T21:21:22.368 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:22.364+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423450 2026-02-20T21:21:22.369 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:22.364+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423930 2026-02-20T21:21:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 54167552 bytes 2026-02-20T21:21:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 99392512 bytes 2026-02-20T21:21:23.213 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:23.210+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:21:23.215 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:23.212+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:21:23.215 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:23.212+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:21:23.216 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:23.213+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:21:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 212545536 bytes 2026-02-20T21:21:24.851 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:24.848+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:21:25.321 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:25.318+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:21:25.322 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:25.318+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:21:25.322 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:25.319+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:21:25.561 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:25.557+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:25.562 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:25.559+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:25.564 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:25.560+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:25.564 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:25.560+0000 7fc9d03ed640 -1 reset not still connected to 0x5576df3755f0 2026-02-20T21:21:25.564 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:25.560+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:21:25.564 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:25.560+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:21:25.564 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:25.560+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:25.564 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:21:25.560+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb380 2026-02-20T21:21:25.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 275219456 bytes 2026-02-20T21:21:26.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 325537792 bytes 2026-02-20T21:21:26.966 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:26.963+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:21:26.966 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:26.963+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:21:26.966 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:26.963+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:21:26.966 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:26.963+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:21:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 349791232 bytes 2026-02-20T21:21:27.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 390640640 bytes 2026-02-20T21:21:27.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 409701376 bytes 2026-02-20T21:21:28.154 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:28.150+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:21:28.154 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:28.150+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:21:28.154 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:28.150+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:21:28.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 438690816 bytes 2026-02-20T21:21:28.627 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:28.624+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:21:28.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 464472064 bytes 2026-02-20T21:21:29.288 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:29.284+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:21:29.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 521425920 bytes 2026-02-20T21:21:30.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 570003456 bytes 2026-02-20T21:21:30.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 622236672 bytes 2026-02-20T21:21:32.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 683300864 bytes 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_15 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_15 journal 128M 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_15 128M 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:32.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:21:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:21:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:21:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:21:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:32.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_15 2026-02-20T21:21:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:21:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:21:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:21:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:21:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:21:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_15 2026-02-20T21:21:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:21:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:21:32.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-20T21:21:32.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:21:32.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_15 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:21:32.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 23601152 bytes 2026-02-20T21:21:33.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 74827776 bytes 2026-02-20T21:21:33.928 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:33.925+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:21:33.929 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:33.926+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:21:33.929 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:33.926+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:21:34.728 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:34.724+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:21:34.728 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:34.724+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:21:34.728 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:34.724+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:21:34.728 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:34.724+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:21:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 119188480 bytes 2026-02-20T21:21:35.451 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:35.447+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:21:35.451 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:35.447+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:21:35.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:35.448+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:21:35.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:35.448+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:21:35.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:35.448+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:21:35.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:35.448+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:21:35.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:35.448+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:21:35.530 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:35.526+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:21:36.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 181447680 bytes 2026-02-20T21:21:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 235953152 bytes 2026-02-20T21:21:37.500 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:37.497+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:21:37.501 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:37.497+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:37.502 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:37.498+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:37.502 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:37.499+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:37.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 285725696 bytes 2026-02-20T21:21:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:21:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:21:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:21:38.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:21:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:21:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_16 2026-02-20T21:21:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_16 journal 128M 2026-02-20T21:21:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:21:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:38.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:21:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:21:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_16 128M 2026-02-20T21:21:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:21:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:21:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:21:38.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:21:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:38.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_16 2026-02-20T21:21:38.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:21:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:21:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:21:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:21:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:21:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_16 2026-02-20T21:21:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:38.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:38.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:21:38.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:21:38.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-20T21:21:38.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:21:38.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_16 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:21:38.493 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:38.489+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:21:38.493 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:38.489+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:21:38.493 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:38.489+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:21:38.493 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:38.489+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:21:38.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 33177600 bytes 2026-02-20T21:21:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 58913792 bytes 2026-02-20T21:21:39.276 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:39.273+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:21:39.277 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:39.274+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:21:39.278 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:39.274+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:21:39.279 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:39.275+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:21:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 164722688 bytes 2026-02-20T21:21:41.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 228290560 bytes 2026-02-20T21:21:42.649 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:21:42.645+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:21:42.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 310758400 bytes 2026-02-20T21:21:42.712 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:42.708+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:21:42.712 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:42.708+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:21:42.712 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:42.708+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:21:42.826 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:42.822+0000 7fc9d03ed640 -1 reset not still connected to 0x5576df3755f0 2026-02-20T21:21:42.827 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:42.823+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:21:42.827 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:42.823+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:21:42.827 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:42.823+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:21:42.828 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:42.823+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:21:42.828 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:42.823+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:42.828 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:42.823+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:42.828 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:42.823+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:42.990 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:42.987+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:21:43.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 341422080 bytes 2026-02-20T21:21:43.936 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:43.933+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb32b0 2026-02-20T21:21:44.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 377145344 bytes 2026-02-20T21:21:44.806 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:44.802+0000 7fd67aee6640 -1 reset not still connected to 0x55daae439520 2026-02-20T21:21:44.806 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:44.802+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f8340 2026-02-20T21:21:44.807 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:44.803+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423450 2026-02-20T21:21:44.807 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:44.803+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423930 2026-02-20T21:21:44.807 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:44.803+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511c270 2026-02-20T21:21:45.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 458343424 bytes 2026-02-20T21:21:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 520349696 bytes 2026-02-20T21:21:45.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 536685568 bytes 2026-02-20T21:21:46.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 565269504 bytes 2026-02-20T21:21:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 594967552 bytes 2026-02-20T21:21:47.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 608596992 bytes 2026-02-20T21:21:47.685 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:47.682+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:21:47.686 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:47.682+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:47.686 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:47.682+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:47.687 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:47.683+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:48.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 664536064 bytes 2026-02-20T21:21:48.522 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:48.519+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:21:48.523 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:48.520+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:21:48.699 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:21:48.697+0000 7f03c8435640 -1 reset not still connected to 0x55afcdaf6dd0 2026-02-20T21:21:48.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 699758592 bytes 2026-02-20T21:21:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 768021504 bytes 2026-02-20T21:21:50.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 795145216 bytes 2026-02-20T21:21:50.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 824178688 bytes 2026-02-20T21:21:51.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:21:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:21:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:21:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:21:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:21:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_17 2026-02-20T21:21:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_17 journal 128M 2026-02-20T21:21:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:21:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:21:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:21:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_17 128M 2026-02-20T21:21:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:21:51.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:21:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:21:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:21:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:21:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_17 2026-02-20T21:21:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:21:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:21:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:21:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:21:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:21:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_17 2026-02-20T21:21:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:21:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:21:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:21:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:21:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-20T21:21:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:21:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_17 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:21:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 8357888 bytes 2026-02-20T21:21:51.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 20925440 bytes 2026-02-20T21:21:52.069 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:52.065+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:52.070 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:52.066+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:52.070 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:52.066+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 46183424 bytes 2026-02-20T21:21:52.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 56968192 bytes 2026-02-20T21:21:52.377 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:52.374+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:21:52.377 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:52.374+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:21:52.377 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:52.374+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:21:52.377 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:52.374+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:21:52.377 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:52.374+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:21:52.378 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:52.374+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:21:52.378 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:21:52.374+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:21:52.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 86339584 bytes 2026-02-20T21:21:53.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 112791552 bytes 2026-02-20T21:21:53.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 131058688 bytes 2026-02-20T21:21:54.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 160819200 bytes 2026-02-20T21:21:55.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 231492608 bytes 2026-02-20T21:21:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 260520960 bytes 2026-02-20T21:21:56.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 279464960 bytes 2026-02-20T21:21:57.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 361193472 bytes 2026-02-20T21:21:59.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 495771648 bytes 2026-02-20T21:21:59.780 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:59.776+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423450 2026-02-20T21:21:59.780 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:59.776+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423930 2026-02-20T21:21:59.780 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:59.776+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511c270 2026-02-20T21:21:59.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 518274048 bytes 2026-02-20T21:21:59.781 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:59.778+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d520 2026-02-20T21:21:59.781 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:59.778+0000 7fd67aee6640 -1 reset not still connected to 0x55daae439ad0 2026-02-20T21:21:59.781 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:21:59.778+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:21:59.861 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:21:59.858+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:22:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 543352832 bytes 2026-02-20T21:22:00.416 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:00.412+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:22:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:22:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:22:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:22:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:22:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:22:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_18 2026-02-20T21:22:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_18 journal 128M 2026-02-20T21:22:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:01.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:22:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:22:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:22:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_18 128M 2026-02-20T21:22:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:22:01.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:01.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:22:01.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:22:01.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:22:01.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:01.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_18 2026-02-20T21:22:01.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:22:01.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:01.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:22:01.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:22:01.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:22:01.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:22:01.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_18 2026-02-20T21:22:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:22:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:22:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=5 2026-02-20T21:22:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:22:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_18 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:22:01.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 5056512 bytes 2026-02-20T21:22:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 60278784 bytes 2026-02-20T21:22:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 137944064 bytes 2026-02-20T21:22:04.521 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:04.518+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:22:04.522 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:04.518+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:22:05.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 218723328 bytes 2026-02-20T21:22:06.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 270164992 bytes 2026-02-20T21:22:06.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:22:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:22:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:22:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:22:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:22:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_19 2026-02-20T21:22:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_19 journal 128M 2026-02-20T21:22:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:06.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:22:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:22:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:22:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_19 128M 2026-02-20T21:22:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:22:06.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:22:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:22:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:22:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:06.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_19 2026-02-20T21:22:06.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_19 2026-02-20T21:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:22:06.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:22:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-20T21:22:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:22:06.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_19 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:22:06.683 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:06.680+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:22:06.684 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:06.680+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:22:06.684 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:06.680+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:22:06.684 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:06.680+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:22:06.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 27493376 bytes 2026-02-20T21:22:07.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 67746816 bytes 2026-02-20T21:22:08.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 147236864 bytes 2026-02-20T21:22:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 173854720 bytes 2026-02-20T21:22:10.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 254407680 bytes 2026-02-20T21:22:10.661 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:10.658+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:22:10.694 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:10.691+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:22:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 265250816 bytes 2026-02-20T21:22:12.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 346830848 bytes 2026-02-20T21:22:12.582 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:12.578+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:22:12.582 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:12.579+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:22:13.181 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:13.178+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:22:13.182 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:13.178+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:22:13.182 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:13.179+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:22:13.183 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:13.179+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:22:13.183 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:13.179+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:22:13.183 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:13.179+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:22:13.184 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:13.180+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:22:13.184 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:13.180+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:22:13.184 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:13.180+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:22:13.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 384747520 bytes 2026-02-20T21:22:13.354 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:13.351+0000 7f413d8ae640 -1 reset not still connected to 0x55d271027450 2026-02-20T21:22:13.355 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:13.351+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb380 2026-02-20T21:22:13.356 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:13.352+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dda00 2026-02-20T21:22:13.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 421711872 bytes 2026-02-20T21:22:14.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:22:14.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:22:14.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:22:14.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:22:14.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:22:14.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_20 2026-02-20T21:22:14.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_20 journal 128M 2026-02-20T21:22:14.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:14.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:14.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:14.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:14.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:22:14.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:22:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:22:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_20 128M 2026-02-20T21:22:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:22:14.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:22:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:22:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:22:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:14.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_20 2026-02-20T21:22:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:22:14.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:22:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:22:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:22:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:22:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_20 2026-02-20T21:22:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:22:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:22:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=11 2026-02-20T21:22:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:22:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_20 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:22:15.208 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:15.205+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:22:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 40916992 bytes 2026-02-20T21:22:15.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 89314304 bytes 2026-02-20T21:22:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 89937920 bytes 2026-02-20T21:22:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 126483456 bytes 2026-02-20T21:22:16.820 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:16.816+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:22:16.824 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:16.821+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:22:16.824 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:16.821+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:22:17.302 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:17.298+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:22:17.302 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:17.298+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:22:18.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 182482944 bytes 2026-02-20T21:22:18.872 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:18.868+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:22:18.873 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:18.870+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:22:18.874 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:18.870+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:22:18.874 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:18.870+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:22:18.874 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:18.870+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:22:18.874 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:18.870+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:22:19.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 276677632 bytes 2026-02-20T21:22:20.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 287527936 bytes 2026-02-20T21:22:20.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 317636608 bytes 2026-02-20T21:22:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 420371456 bytes 2026-02-20T21:22:22.686 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-20T21:22:22.686 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-20T21:22:22.686 INFO:tasks.rbd_mirror_thrash:waiting for 7.101968440717362 secs before reviving daemons 2026-02-20T21:22:22.687 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:22:22.684+0000 7f200537c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 90267) UID: 1000 2026-02-20T21:22:22.852 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:22.848+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:22:22.852 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:22.848+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:22:23.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 488001536 bytes 2026-02-20T21:22:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 500005888 bytes 2026-02-20T21:22:23.730 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:23.726+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:22:23.730 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:23.726+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:22:23.730 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:23.726+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:22:23.732 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:23.729+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:22:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 542097408 bytes 2026-02-20T21:22:24.345 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:24.341+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:22:24.346 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:24.343+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:22:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 574670848 bytes 2026-02-20T21:22:25.211 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:25.207+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9110 2026-02-20T21:22:25.212 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:25.208+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:22:25.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:22:25.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:22:25.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:22:25.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:22:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:22:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_21 2026-02-20T21:22:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_21 journal 128M 2026-02-20T21:22:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:22:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:22:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:22:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_21 128M 2026-02-20T21:22:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:22:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:22:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:22:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:22:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_21 2026-02-20T21:22:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:22:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:22:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:22:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:22:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:22:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_21 2026-02-20T21:22:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:22:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:22:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=5 2026-02-20T21:22:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:22:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_21 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:22:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 40844288 bytes 2026-02-20T21:22:26.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 57491456 bytes 2026-02-20T21:22:26.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 61485056 bytes 2026-02-20T21:22:28.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 141696000 bytes 2026-02-20T21:22:28.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 189231104 bytes 2026-02-20T21:22:29.685 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:29.682+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:22:29.687 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:29.683+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:22:29.687 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:29.683+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:22:29.789 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-20T21:22:29.789 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:22:29.789 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-20T21:22:29.789 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T21:22:29.790 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T21:22:29.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T21:22:29.792 INFO:tasks.rbd_mirror_thrash:waiting for 62.82887480606881 secs before thrashing 2026-02-20T21:22:29.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 240564224 bytes 2026-02-20T21:22:29.981 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:22:29.979+0000 7f03c8435640 -1 reset not still connected to 0x55afcdaf61a0 2026-02-20T21:22:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 241192960 bytes 2026-02-20T21:22:30.154 INFO:tasks.ceph.cluster2.osd.1.trial163.stderr:2026-02-20T21:22:30.151+0000 7fa51ae2f640 -1 bdev(0x5560efe7b400 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 2 2026-02-20T21:22:30.155 INFO:tasks.ceph.cluster2.osd.1.trial163.stderr:2026-02-20T21:22:30.153+0000 7fa518e2b640 -1 bdev(0x5560efe7b400 /var/lib/ceph/osd/cluster2-1/block) aio_submit retries 4 2026-02-20T21:22:30.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 258010112 bytes 2026-02-20T21:22:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:22:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:22:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:22:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:22:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:22:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_22 2026-02-20T21:22:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_22 journal 128M 2026-02-20T21:22:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:22:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:22:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:22:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_22 128M 2026-02-20T21:22:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:22:30.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:22:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:22:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:22:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_22 2026-02-20T21:22:30.483 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.31108985284911694) > weight (0.25) 2026-02-20T21:22:30.484 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9922754545074285) > weight (0.25) 2026-02-20T21:22:30.484 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-20T21:22:30.484 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-20T21:22:30.484 INFO:tasks.rbd_mirror_thrash:waiting for 8.247795599806722 secs before reviving daemons 2026-02-20T21:22:30.485 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:22:30.483+0000 7f756d7cb640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 49316) UID: 1000 2026-02-20T21:22:30.485 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:22:30.483+0000 7f75667bd640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x5587b1064a80 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-20T21:22:30.485 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:22:30.483+0000 7f75667bd640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x5587b1064a80 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-20T21:22:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:22:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:22:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:22:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:22:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:22:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_22 2026-02-20T21:22:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:22:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:22:30.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-20T21:22:30.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:22:30.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_22 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:22:31.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 46177280 bytes 2026-02-20T21:22:31.666 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:31.663+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:22:31.667 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:31.663+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:22:31.667 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:31.664+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:22:31.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 70782976 bytes 2026-02-20T21:22:32.364 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:32.360+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:22:32.365 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:32.362+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:22:32.366 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:32.363+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:22:32.727 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:32.723+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:22:32.727 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:32.723+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:22:32.727 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:32.723+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:22:32.727 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:32.723+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:22:32.728 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:32.724+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:22:32.728 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:32.724+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:22:32.774 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:32.769+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc6f70 2026-02-20T21:22:32.775 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:32.769+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:22:32.775 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:32.771+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3143450 2026-02-20T21:22:32.775 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:32.771+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:22:32.775 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:32.771+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:22:32.775 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:32.771+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:22:33.014 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:33.010+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:22:33.014 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:33.010+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:22:33.014 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:33.010+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:22:33.014 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:33.011+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:22:33.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 157084672 bytes 2026-02-20T21:22:34.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 187518976 bytes 2026-02-20T21:22:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 208987136 bytes 2026-02-20T21:22:35.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 217827328 bytes 2026-02-20T21:22:36.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:22:36.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:22:36.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:22:36.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:22:36.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:22:36.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_23 2026-02-20T21:22:36.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_23 journal 128M 2026-02-20T21:22:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:22:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:22:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:22:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_23 128M 2026-02-20T21:22:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:36.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:22:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:22:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:22:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:22:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:36.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_23 2026-02-20T21:22:36.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:22:36.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:36.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:22:36.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:22:36.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:22:36.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:22:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_23 2026-02-20T21:22:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:22:36.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:22:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=11 2026-02-20T21:22:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:22:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_23 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:22:37.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 43922432 bytes 2026-02-20T21:22:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 81820672 bytes 2026-02-20T21:22:37.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 93272064 bytes 2026-02-20T21:22:38.733 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-20T21:22:38.733 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:22:38.733 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-20T21:22:38.733 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-20T21:22:38.734 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-20T21:22:38.736 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-20T21:22:38.736 INFO:tasks.rbd_mirror_thrash:waiting for 89.43590979813679 secs before thrashing 2026-02-20T21:22:38.848 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:38.845+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc920d0 2026-02-20T21:22:38.849 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:38.845+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:22:38.849 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:38.845+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:22:38.850 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:38.846+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:22:38.850 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:38.847+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:22:38.851 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:38.847+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:22:38.851 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:38.847+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:22:38.852 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:38.848+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:22:38.852 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:38.848+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:22:38.852 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:38.848+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:22:39.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 165492736 bytes 2026-02-20T21:22:39.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 198401024 bytes 2026-02-20T21:22:40.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 215560192 bytes 2026-02-20T21:22:40.370 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:40.367+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:22:40.371 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:40.367+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:22:40.372 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:40.369+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:22:40.372 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:40.369+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:22:40.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 268941312 bytes 2026-02-20T21:22:40.942 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:40.938+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:22:40.942 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:40.938+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:22:40.943 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:40.940+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:22:40.943 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:40.940+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:22:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 329446400 bytes 2026-02-20T21:22:42.238 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:42.234+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:22:42.239 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:42.236+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:22:42.643 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:42.640+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:22:42.644 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:42.641+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:22:42.645 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:42.641+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:22:42.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 371710976 bytes 2026-02-20T21:22:43.058 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:43.055+0000 7fd67aee6640 -1 reset not still connected to 0x55daae439ad0 2026-02-20T21:22:43.059 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:43.056+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:22:43.059 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:43.056+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:22:43.059 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:43.056+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:22:43.504 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:43.501+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:22:43.505 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:43.501+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:22:43.505 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:43.501+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:22:43.506 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:43.502+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:22:43.506 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:43.502+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:22:43.506 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:43.502+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:22:43.507 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:43.503+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:22:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 445441024 bytes 2026-02-20T21:22:44.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 504818688 bytes 2026-02-20T21:22:44.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 515952640 bytes 2026-02-20T21:22:45.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 529305600 bytes 2026-02-20T21:22:45.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 543418368 bytes 2026-02-20T21:22:46.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 645000192 bytes 2026-02-20T21:22:47.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:22:47.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:22:47.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:22:47.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:22:47.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:22:47.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_24 2026-02-20T21:22:47.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_24 journal 128M 2026-02-20T21:22:47.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:47.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:47.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:47.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:47.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:22:47.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:47.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:22:47.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:22:47.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:47.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_24 128M 2026-02-20T21:22:47.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:47.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:22:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:22:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:22:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:22:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_24 2026-02-20T21:22:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:22:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:22:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:22:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:22:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:22:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_24 2026-02-20T21:22:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:22:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:22:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-20T21:22:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:22:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_24 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:22:48.176 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:48.172+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:22:48.176 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:48.172+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:22:48.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 53710848 bytes 2026-02-20T21:22:48.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 87996416 bytes 2026-02-20T21:22:48.999 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:48.996+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:22:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 113893376 bytes 2026-02-20T21:22:49.585 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:49.581+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:22:49.586 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:49.582+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:22:49.587 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:49.584+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:22:49.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 148430848 bytes 2026-02-20T21:22:50.040 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:50.037+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:22:50.040 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:50.037+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:22:50.042 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:50.038+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:22:50.042 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:50.039+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:22:51.091 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:51.087+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:22:51.092 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:51.087+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:22:51.092 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:51.087+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:22:51.092 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:51.087+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:22:51.092 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:51.089+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:22:51.092 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:51.089+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:22:51.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 189946880 bytes 2026-02-20T21:22:51.812 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:51.809+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:22:51.813 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:51.809+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:22:52.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 235645952 bytes 2026-02-20T21:22:52.182 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:52.179+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:22:52.183 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:52.180+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:22:52.183 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:52.180+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:22:52.433 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:52.430+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:22:52.575 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:52.572+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:22:52.576 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:52.573+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:22:52.576 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:52.573+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:22:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 249535488 bytes 2026-02-20T21:22:52.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 266956800 bytes 2026-02-20T21:22:54.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 331382784 bytes 2026-02-20T21:22:54.315 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:54.311+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:22:54.437 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:54.433+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:22:54.438 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:54.434+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:22:54.438 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:22:54.435+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_25 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_25 journal 128M 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:22:54.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:22:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:22:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_25 128M 2026-02-20T21:22:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:22:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:22:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:22:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:22:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:22:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_25 2026-02-20T21:22:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:22:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:22:54.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:22:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:22:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:22:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_25 2026-02-20T21:22:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:22:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:22:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:22:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:22:54.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-20T21:22:54.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:22:54.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_25 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:22:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 57178112 bytes 2026-02-20T21:22:56.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 88947712 bytes 2026-02-20T21:22:56.277 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:22:56.273+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:22:56.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 101392384 bytes 2026-02-20T21:22:56.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 113079296 bytes 2026-02-20T21:22:56.833 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:56.830+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:22:56.834 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:56.830+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:22:56.834 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:56.830+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:22:56.834 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:56.831+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:22:56.834 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:56.831+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:22:56.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 135852032 bytes 2026-02-20T21:22:58.596 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:58.592+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:22:58.596 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:58.592+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:22:58.596 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:58.592+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:22:58.597 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:58.594+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:22:58.597 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:58.594+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:22:59.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 204915712 bytes 2026-02-20T21:22:59.629 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:59.625+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:22:59.629 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:59.625+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:22:59.630 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:59.626+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:22:59.630 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:22:59.626+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:23:00.148 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:00.145+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:23:00.149 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:00.146+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3143450 2026-02-20T21:23:00.149 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:00.146+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:23:00.149 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:00.146+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:23:00.150 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:00.146+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:23:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 267168768 bytes 2026-02-20T21:23:00.739 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:00.736+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc920d0 2026-02-20T21:23:00.740 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:00.736+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:23:00.741 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:00.737+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:23:00.741 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:00.737+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:23:00.741 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:00.737+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:23:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 275998720 bytes 2026-02-20T21:23:00.935 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:00.931+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:23:00.937 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:00.933+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:23:00.937 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:00.933+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:23:00.937 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:00.934+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:23:01.097 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:01.093+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:23:01.097 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:01.093+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:23:01.097 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:01.093+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:01.098 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:01.094+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:01.098 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:01.094+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:01.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 318325760 bytes 2026-02-20T21:23:01.530 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:01.526+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:23:01.530 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:01.526+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:23:01.530 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:01.526+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:23:01.530 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:01.526+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:23:01.530 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:01.526+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:23:01.530 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:01.526+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:23:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 327624704 bytes 2026-02-20T21:23:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 346576896 bytes 2026-02-20T21:23:01.861 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:01.857+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:23:01.861 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:01.858+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:23:01.862 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:01.858+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:23:02.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 368455680 bytes 2026-02-20T21:23:02.512 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:02.508+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:23:02.514 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:02.510+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:23:02.514 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:02.510+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:23:02.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:23:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:23:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:23:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:23:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:23:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_26 2026-02-20T21:23:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_26 journal 128M 2026-02-20T21:23:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:23:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:23:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:23:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_26 128M 2026-02-20T21:23:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:23:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:23:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:23:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:23:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_26 2026-02-20T21:23:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:23:02.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:23:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:23:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:23:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:23:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_26 2026-02-20T21:23:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:23:02.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:23:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-20T21:23:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:23:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_26 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:23:03.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 49216512 bytes 2026-02-20T21:23:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 83668992 bytes 2026-02-20T21:23:04.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 114954240 bytes 2026-02-20T21:23:05.883 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:05.880+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:06.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 156814336 bytes 2026-02-20T21:23:06.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 157430784 bytes 2026-02-20T21:23:06.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 170344448 bytes 2026-02-20T21:23:07.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 231117824 bytes 2026-02-20T21:23:08.216 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:08.212+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:23:08.217 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:08.213+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:23:08.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 271738880 bytes 2026-02-20T21:23:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 297482240 bytes 2026-02-20T21:23:09.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 298086400 bytes 2026-02-20T21:23:09.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 317253632 bytes 2026-02-20T21:23:09.889 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:09.886+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:23:09.890 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:09.886+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 371664896 bytes 2026-02-20T21:23:10.449 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:10.445+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:23:10.449 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:10.445+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:23:10.450 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:10.445+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:23:10.450 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:10.445+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:23:10.450 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:10.445+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:23:10.450 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:10.445+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:23:10.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 403443712 bytes 2026-02-20T21:23:11.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 442045440 bytes 2026-02-20T21:23:11.692 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:11.688+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:23:11.724 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:11.721+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:23:12.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 465511424 bytes 2026-02-20T21:23:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 524588032 bytes 2026-02-20T21:23:12.862 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:12.859+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:23:12.862 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:12.859+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:23:12.865 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:12.860+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:23:12.865 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:12.860+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:23:12.865 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:12.860+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:23:12.866 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:12.861+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:23:13.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 556724224 bytes 2026-02-20T21:23:13.520 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:13.517+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:23:13.521 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:13.517+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:23:13.523 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:13.518+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:23:13.523 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:13.518+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:23:13.523 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:13.518+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:23:13.523 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:13.518+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:23:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 574108672 bytes 2026-02-20T21:23:13.946 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:13.942+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:23:13.946 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:13.942+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:23:13.946 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:13.942+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:23:13.946 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:13.942+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:23:13.946 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:13.942+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:23:13.946 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:13.942+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:23:13.948 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:13.945+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:23:13.948 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:13.945+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:23:13.948 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:13.945+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:23:13.948 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:13.945+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:23:14.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 620100608 bytes 2026-02-20T21:23:14.987 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:14.983+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:23:14.987 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:14.984+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:23:14.987 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:14.984+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:23:14.987 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:14.984+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:23:14.988 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:14.985+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:23:14.988 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:14.985+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:23:15.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 682231808 bytes 2026-02-20T21:23:15.379 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:15.376+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:23:15.380 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:15.376+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:23:15.380 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:15.376+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:23:15.380 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:15.376+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:23:15.380 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:15.376+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:23:15.380 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:15.376+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:23:15.382 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:15.379+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:23:15.382 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:15.379+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:23:15.382 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:15.379+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:23:15.383 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:15.379+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:23:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 685736960 bytes 2026-02-20T21:23:15.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2200 IOs, 716776448 bytes 2026-02-20T21:23:15.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:23:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:23:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:23:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:23:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:23:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_27 2026-02-20T21:23:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_27 journal 128M 2026-02-20T21:23:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:23:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:23:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:23:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_27 128M 2026-02-20T21:23:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:23:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:23:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:23:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:23:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_27 2026-02-20T21:23:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:23:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:23:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:23:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:23:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:23:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_27 2026-02-20T21:23:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:23:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:23:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-20T21:23:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:23:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_27 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:23:16.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 6726656 bytes 2026-02-20T21:23:16.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 35147776 bytes 2026-02-20T21:23:16.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 47887360 bytes 2026-02-20T21:23:17.063 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:17.059+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:17.121 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:17.118+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:17.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 68936704 bytes 2026-02-20T21:23:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 112696320 bytes 2026-02-20T21:23:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 120320000 bytes 2026-02-20T21:23:19.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 144942080 bytes 2026-02-20T21:23:19.418 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.414+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:23:19.418 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.414+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423ad0 2026-02-20T21:23:19.419 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.415+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:23:19.419 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:23:19.417+0000 7f03c8435640 -1 reset not still connected to 0x55afbd483520 2026-02-20T21:23:19.420 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:23:19.417+0000 7f03c8435640 -1 reset not still connected to 0x55afc2052340 2026-02-20T21:23:19.420 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.417+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:23:19.422 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.419+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:23:19.422 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.419+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:19.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.421+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:19.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.421+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:19.638 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.635+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:23:19.661 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.657+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:23:19.661 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.657+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423ad0 2026-02-20T21:23:19.663 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.659+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:23:19.663 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.659+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:19.663 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.659+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:19.663 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:19.659+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:19.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 166177792 bytes 2026-02-20T21:23:20.220 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:20.216+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:23:20.221 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:20.217+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:23:20.221 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:20.217+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:23:20.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 215808000 bytes 2026-02-20T21:23:20.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 219222016 bytes 2026-02-20T21:23:21.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 241736704 bytes 2026-02-20T21:23:21.656 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:21.652+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:23:21.657 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:21.653+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:23:21.657 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:21.653+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:23:21.658 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:21.653+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:21.658 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:21.654+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:21.658 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:21.654+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:21.921 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:21.917+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:23:21.921 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:21.917+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:23:22.294 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:23:22.292+0000 7f03c8435640 -1 reset not still connected to 0x55af9f645ee0 2026-02-20T21:23:22.298 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:23:22.296+0000 7f03c8435640 -1 reset not still connected to 0x55afabb112b0 2026-02-20T21:23:22.298 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:23:22.296+0000 7f03c8435640 -1 reset not still connected to 0x55afaec9c5b0 2026-02-20T21:23:22.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 288890880 bytes 2026-02-20T21:23:22.768 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:22.764+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:23:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 341954560 bytes 2026-02-20T21:23:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:23:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:23:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:23:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:23:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:23:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_28 2026-02-20T21:23:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_28 journal 128M 2026-02-20T21:23:23.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:23:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:23:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:23:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:23.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_28 128M 2026-02-20T21:23:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:23:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:23:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:23:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:23:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:23.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_28 2026-02-20T21:23:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:23:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:23:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:23:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:23:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:23:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_28 2026-02-20T21:23:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:23:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:23:23.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=9 2026-02-20T21:23:23.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:23:23.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_28 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:23:23.196 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:23.193+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:23:23.198 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:23.194+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:23:23.198 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:23.195+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:23:23.199 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:23.196+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:23:23.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 26509312 bytes 2026-02-20T21:23:24.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 103583744 bytes 2026-02-20T21:23:24.985 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:24.982+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:23:24.985 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:24.982+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:24.986 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:24.983+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:24.987 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:24.983+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 145957888 bytes 2026-02-20T21:23:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 146585600 bytes 2026-02-20T21:23:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 196531200 bytes 2026-02-20T21:23:27.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 266134528 bytes 2026-02-20T21:23:27.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 269367296 bytes 2026-02-20T21:23:28.242 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.238+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:28.243 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.238+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:28.243 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.238+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d1e0 2026-02-20T21:23:28.243 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.238+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:23:28.244 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.239+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423ad0 2026-02-20T21:23:28.244 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.239+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:23:28.244 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.239+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:23:28.244 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.239+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:23:28.244 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.239+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:28.244 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.239+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:28.244 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.239+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:28.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 299244544 bytes 2026-02-20T21:23:28.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 305059840 bytes 2026-02-20T21:23:28.927 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.923+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:28.928 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.925+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:28.928 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:28.925+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 315496448 bytes 2026-02-20T21:23:29.158 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.154+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:23:29.159 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.154+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:23:29.159 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.154+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:23:29.159 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.154+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:23:29.159 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.155+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:23:29.420 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.417+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:23:29.804 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.800+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:23:29.828 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.824+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:23:29.828 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.824+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:23:29.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 344562688 bytes 2026-02-20T21:23:29.905 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.901+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351fee0 2026-02-20T21:23:29.906 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.903+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:23:29.906 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.903+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:23:29.906 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:29.903+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:23:30.101 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:30.097+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:23:30.101 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:30.098+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:23:30.141 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:30.138+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:23:30.141 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:30.138+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:23:30.141 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:30.138+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:23:30.142 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:30.139+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:23:30.420 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:30.416+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:23:30.420 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:30.416+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:23:30.420 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:30.416+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:23:30.420 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:30.416+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:23:30.421 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:30.416+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:23:30.421 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:30.417+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:23:30.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 374567936 bytes 2026-02-20T21:23:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 409419776 bytes 2026-02-20T21:23:31.213 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:31.209+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:31.994 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:31.990+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351fee0 2026-02-20T21:23:31.995 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:31.991+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:23:31.995 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:31.991+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:23:31.995 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:31.991+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:23:32.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_29 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_29 journal 128M 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_29 128M 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:32.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_29 2026-02-20T21:23:32.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:23:32.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:32.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:23:32.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:23:32.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:23:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:23:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_29 2026-02-20T21:23:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:23:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:23:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-20T21:23:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:23:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_29 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:23:32.111 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:32.108+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:23:32.113 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:32.110+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:23:32.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:32.110+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:23:32.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:32.110+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:23:32.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:32.110+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:23:32.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:32.110+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:23:32.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:32.110+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:23:32.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:32.110+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:23:32.559 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:32.555+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:23:32.562 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:32.558+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:23:32.620 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:32.617+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:23:32.622 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.2633247224595453) > weight (0.25) 2026-02-20T21:23:32.622 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.4270718536339534) > weight (0.25) 2026-02-20T21:23:32.622 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-20T21:23:32.623 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-20T21:23:32.623 INFO:tasks.rbd_mirror_thrash:waiting for 0.03951619680274132 secs before reviving daemons 2026-02-20T21:23:32.623 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:23:32.620+0000 7f6191dbe640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 84179) UID: 1000 2026-02-20T21:23:32.663 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-20T21:23:32.663 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:23:32.756 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:32.753+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:23:32.757 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:32.753+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:23:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 63119360 bytes 2026-02-20T21:23:33.646 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.643+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:23:33.647 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.644+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:23:33.647 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.644+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:23:33.648 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.644+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:23:33.648 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.644+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:23:33.649 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.646+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:23:33.667 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.663+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:23:33.668 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.665+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:23:33.668 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.665+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:23:33.668 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.665+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:23:33.674 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.670+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:23:33.674 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.670+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:23:33.675 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.671+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:23:33.675 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:33.672+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:23:34.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 101917696 bytes 2026-02-20T21:23:34.915 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:34.911+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:23:34.915 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:34.911+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:23:34.915 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:34.911+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:23:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 109261824 bytes 2026-02-20T21:23:35.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 156658688 bytes 2026-02-20T21:23:36.123 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.119+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351fee0 2026-02-20T21:23:36.124 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.121+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e89751e0 2026-02-20T21:23:36.125 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.121+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:23:36.125 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.121+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:23:36.127 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.124+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:23:36.421 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.418+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:23:36.423 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.419+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:23:36.423 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.419+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:23:36.423 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.419+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:23:36.426 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.422+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:23:36.426 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.422+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:23:36.428 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:23:36.428 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:23:36.428 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:23:36.429 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:23:36.429 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x5576df375450 2026-02-20T21:23:36.429 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x5576df3755f0 2026-02-20T21:23:36.429 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:23:36.429 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:23:36.429 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:23:36.429 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:23:36.429 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:23:36.429 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:23:36.429 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:36.424+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:23:36.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 212021248 bytes 2026-02-20T21:23:36.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 212617216 bytes 2026-02-20T21:23:37.665 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:37.661+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:23:37.665 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:37.661+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:23:37.666 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:37.663+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:23:37.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 254817280 bytes 2026-02-20T21:23:38.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 269526016 bytes 2026-02-20T21:23:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 270161920 bytes 2026-02-20T21:23:38.764 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-20T21:23:38.764 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-20T21:23:38.764 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-20T21:23:38.767 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-20T21:23:38.767 INFO:tasks.rbd_mirror_thrash:waiting for 78.2047730523556 secs before thrashing 2026-02-20T21:23:38.936 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:38.932+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:38.937 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:38.934+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:38.937 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:38.934+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:39.071 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:39.067+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:23:39.071 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:39.067+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:23:39.071 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:39.067+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:23:39.071 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:39.067+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:23:39.071 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:39.067+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:23:39.073 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:39.070+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:23:39.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 330707968 bytes 2026-02-20T21:23:39.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 415887360 bytes 2026-02-20T21:23:41.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 494581760 bytes 2026-02-20T21:23:41.760 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:41.757+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:23:42.081 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:42.077+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:42.082 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:42.079+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:42.082 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:42.079+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:23:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:23:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:23:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:23:42.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:23:42.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_30 2026-02-20T21:23:42.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_30 journal 128M 2026-02-20T21:23:42.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:42.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:42.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:42.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:42.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:23:42.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:42.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:23:42.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:23:42.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:42.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_30 128M 2026-02-20T21:23:42.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:42.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:42.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:42.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:42.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:23:42.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:42.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:23:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:23:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:23:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:42.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_30 2026-02-20T21:23:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:23:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:23:42.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:23:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:23:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:23:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_30 2026-02-20T21:23:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:23:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:23:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=9 2026-02-20T21:23:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:23:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_30 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:23:42.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 36655104 bytes 2026-02-20T21:23:43.460 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:43.456+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:23:43.462 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:43.458+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:23:43.462 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:43.458+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:23:43.462 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:43.458+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:23:43.462 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:43.458+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:23:43.462 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:43.458+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:23:43.462 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:43.458+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:23:43.462 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:43.458+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:23:43.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 139399168 bytes 2026-02-20T21:23:44.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:44.243+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:23:44.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:44.243+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:44.248 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:44.244+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:44.248 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:44.244+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:44.543 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:44.539+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:23:44.545 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:44.542+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:23:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 157761536 bytes 2026-02-20T21:23:45.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 183124992 bytes 2026-02-20T21:23:45.638 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:45.634+0000 7fd67aee6640 -1 reset not still connected to 0x55daae439ad0 2026-02-20T21:23:45.638 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:45.634+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:23:45.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:45.636+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:23:45.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:45.636+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:23:45.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:45.636+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:23:45.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 217320448 bytes 2026-02-20T21:23:46.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 236228608 bytes 2026-02-20T21:23:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 274143232 bytes 2026-02-20T21:23:47.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 274749440 bytes 2026-02-20T21:23:47.261 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:47.258+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:23:47.363 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:47.359+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:23:47.363 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:47.359+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:23:47.363 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:47.359+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:23:47.363 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:47.359+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:47.363 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:47.359+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:47.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 326277120 bytes 2026-02-20T21:23:47.876 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:47.872+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:23:47.876 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:47.872+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:23:47.876 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:47.872+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:23:47.876 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:47.872+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:23:47.877 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:47.872+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:23:47.877 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:47.872+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:23:48.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 410078208 bytes 2026-02-20T21:23:49.558 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:49.555+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:23:49.559 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:49.556+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:23:49.560 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:49.556+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:23:49.560 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:49.556+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:49.561 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:49.558+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:49.561 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:49.558+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:49.828 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:49.824+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:23:49.828 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:49.824+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:23:49.831 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:49.828+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:23:49.892 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:49.889+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:23:49.893 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:49.889+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:23:49.893 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:49.890+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:23:49.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 503809024 bytes 2026-02-20T21:23:50.001 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:49.997+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:23:50.001 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:49.997+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:23:50.001 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:49.998+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:23:50.002 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:49.998+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:23:50.002 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:49.999+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:23:50.002 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:49.999+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:23:50.648 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:50.644+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:23:50.648 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:50.644+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:50.648 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:50.644+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:50.649 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:50.644+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:50.860 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:50.856+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:23:50.860 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:50.856+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:23:50.860 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:50.856+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:23:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:23:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:23:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:23:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:23:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:23:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_31 2026-02-20T21:23:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_31 journal 128M 2026-02-20T21:23:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:51.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:23:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:23:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:23:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_31 128M 2026-02-20T21:23:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:51.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:51.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:23:51.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:51.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:23:51.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:23:51.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:23:51.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:23:51.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_31 2026-02-20T21:23:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:23:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:23:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:23:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:23:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:23:51.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_31 2026-02-20T21:23:51.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:23:51.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:23:51.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:23:51.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:23:51.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-20T21:23:51.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:23:51.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_31 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:23:51.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 49893376 bytes 2026-02-20T21:23:52.284 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:52.281+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:23:52.285 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:52.281+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:23:52.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 64710656 bytes 2026-02-20T21:23:52.426 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:52.422+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:23:52.426 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:52.422+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:23:52.426 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:52.422+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:23:52.472 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:52.469+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:23:52.473 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:52.469+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:23:52.473 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:52.469+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:23:52.474 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:52.470+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:52.544 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:52.540+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351fee0 2026-02-20T21:23:52.544 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:52.540+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:23:52.546 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:52.542+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:23:52.546 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:52.542+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:23:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 99667968 bytes 2026-02-20T21:23:52.803 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:52.799+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:23:53.274 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:23:53.272+0000 7f03c8435640 -1 reset not still connected to 0x55afb9b1a8f0 2026-02-20T21:23:53.275 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:23:53.272+0000 7f03c8435640 -1 reset not still connected to 0x55afc57e7c70 2026-02-20T21:23:53.275 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:23:53.273+0000 7f03c8435640 -1 reset not still connected to 0x55afc5b0dc70 2026-02-20T21:23:53.276 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:23:53.273+0000 7f03c8435640 -1 reset not still connected to 0x55afce0ee750 2026-02-20T21:23:53.276 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:23:53.273+0000 7f03c8435640 -1 reset not still connected to 0x55afcef5c410 2026-02-20T21:23:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 126200832 bytes 2026-02-20T21:23:53.928 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:53.925+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:23:53.928 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:53.925+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:23:53.929 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:53.926+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:23:53.929 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:53.926+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:53.932 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:53.928+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:53.932 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:53.928+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:23:53.932 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:53.928+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:23:53.932 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:53.928+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:54.125 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:54.121+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:54.126 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:54.123+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:23:54.127 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:54.123+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:23:54.127 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:54.123+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:54.257 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:54.253+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:23:54.257 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:54.253+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:23:54.258 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:54.254+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:23:54.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 175414272 bytes 2026-02-20T21:23:55.502 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:55.498+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:23:55.503 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:55.499+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:23:55.503 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:55.499+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:23:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 230255616 bytes 2026-02-20T21:23:56.193 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:56.190+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:23:56.194 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:56.191+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:23:56.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:56.191+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:23:56.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:56.191+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:23:56.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:56.191+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:23:56.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:56.191+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:23:56.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:56.191+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:56.435 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:56.432+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:23:56.437 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:56.433+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:23:56.437 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:56.433+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:23:56.437 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:56.433+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:23:56.758 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:56.754+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:23:56.758 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:56.754+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:23:56.758 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:56.754+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:23:56.873 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:56.870+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:23:56.874 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:56.870+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:23:56.874 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:56.870+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:23:56.874 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:56.870+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:23:56.875 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:56.871+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:23:56.877 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:56.873+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:23:56.877 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:23:56.873+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:23:57.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 291584000 bytes 2026-02-20T21:23:57.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 312602624 bytes 2026-02-20T21:23:57.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 330901504 bytes 2026-02-20T21:23:58.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 372239360 bytes 2026-02-20T21:23:58.431 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:58.428+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:23:58.433 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:23:58.429+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:23:58.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 416709632 bytes 2026-02-20T21:23:59.089 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:59.086+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:23:59.090 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:59.086+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:23:59.090 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:59.086+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:23:59.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 442140672 bytes 2026-02-20T21:23:59.618 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:59.614+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:23:59.624 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:59.620+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:23:59.624 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:59.620+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:23:59.624 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:23:59.620+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:24:00.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 513233920 bytes 2026-02-20T21:24:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 517244928 bytes 2026-02-20T21:24:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 517851136 bytes 2026-02-20T21:24:01.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 550398976 bytes 2026-02-20T21:24:01.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:24:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:24:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:24:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:24:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:24:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_32 2026-02-20T21:24:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_32 journal 128M 2026-02-20T21:24:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:01.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:24:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:24:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:24:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_32 128M 2026-02-20T21:24:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:24:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:24:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:24:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:24:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_32 2026-02-20T21:24:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:24:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:01.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:24:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:24:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:24:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:24:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_32 2026-02-20T21:24:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:24:01.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:24:01.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-20T21:24:01.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:24:01.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_32 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:24:01.730 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:01.726+0000 7fd67aee6640 -1 reset not still connected to 0x55daae439ad0 2026-02-20T21:24:01.731 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:01.726+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:24:01.731 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:01.726+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:24:01.732 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:01.728+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:24:01.732 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:01.728+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:24:01.970 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:01.966+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:01.972 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:01.968+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:01.972 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:01.968+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:01.972 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:01.968+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:02.287 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:02.283+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:24:02.288 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:02.284+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:24:02.288 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:02.284+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:24:02.288 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:02.284+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:02.288 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:02.284+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:02.289 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:02.285+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 33730560 bytes 2026-02-20T21:24:02.815 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:02.811+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:02.815 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:02.811+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:02.815 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:02.811+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:02.815 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:02.811+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:02.815 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:02.811+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:02.816 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:02.811+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 62243840 bytes 2026-02-20T21:24:03.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 70373376 bytes 2026-02-20T21:24:03.368 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:03.364+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:03.368 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:03.364+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:03.368 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:03.364+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:03.369 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:03.365+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:03.428 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:24:03.424+0000 7f03c8435640 -1 reset not still connected to 0x55afd7660a90 2026-02-20T21:24:03.490 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:03.486+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:03.666 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:03.662+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:24:03.666 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:03.662+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:24:03.667 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:03.664+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:24:03.668 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:03.664+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:24:03.668 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:03.664+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:24:03.852 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:03.848+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:24:03.852 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:03.848+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:03.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 116374528 bytes 2026-02-20T21:24:04.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 139041792 bytes 2026-02-20T21:24:04.471 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:04.467+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:24:04.471 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:04.467+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:04.472 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:04.468+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:04.472 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:04.468+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:04.472 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:04.469+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:04.563 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:04.559+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:04.717 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:04.713+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:24:04.718 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:04.714+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:24:04.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:04.716+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:24:04.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:04.716+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:04.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:04.716+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d040 2026-02-20T21:24:04.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:04.716+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc920d0 2026-02-20T21:24:04.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:04.716+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:24:04.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:04.716+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:24:04.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:04.716+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:04.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 162598912 bytes 2026-02-20T21:24:04.858 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:04.854+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:24:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 199604224 bytes 2026-02-20T21:24:06.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 289133568 bytes 2026-02-20T21:24:06.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 304387072 bytes 2026-02-20T21:24:07.422 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:07.418+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:24:07.422 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:07.418+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:24:07.423 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:07.419+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:24:07.423 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:07.419+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:07.424 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:07.421+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:07.424 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:07.421+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:07.424 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:07.421+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:07.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 343111680 bytes 2026-02-20T21:24:07.536 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:07.533+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:08.173 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9340334200503021) > weight (0.25) 2026-02-20T21:24:08.173 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5577246300037209) > weight (0.25) 2026-02-20T21:24:08.173 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.567805076546792) > weight (0.25) 2026-02-20T21:24:08.173 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.4350466587708024) > weight (0.25) 2026-02-20T21:24:08.173 INFO:tasks.rbd_mirror_thrash:waiting for 103.51072789809054 secs before thrashing 2026-02-20T21:24:08.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:08.191+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:08.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:08.191+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:08.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:08.191+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:08.196 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:08.192+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 381625344 bytes 2026-02-20T21:24:08.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 406801408 bytes 2026-02-20T21:24:08.850 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:08.847+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:08.851 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:08.847+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:08.852 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:08.848+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:08.852 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:08.848+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 419206144 bytes 2026-02-20T21:24:09.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:24:09.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:24:09.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:24:09.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:24:09.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:24:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_33 2026-02-20T21:24:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_33 journal 128M 2026-02-20T21:24:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:24:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:24:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:24:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_33 128M 2026-02-20T21:24:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:24:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:24:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:24:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:24:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:09.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_33 2026-02-20T21:24:09.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:24:09.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:09.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:24:09.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:24:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:24:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:24:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_33 2026-02-20T21:24:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:24:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:24:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=10 2026-02-20T21:24:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:24:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 10s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_33 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:24:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 609280 bytes 2026-02-20T21:24:09.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 12103680 bytes 2026-02-20T21:24:10.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 49491968 bytes 2026-02-20T21:24:10.412 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.407+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:24:10.412 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.407+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:24:10.412 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.408+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:10.412 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.408+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:10.413 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.409+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:10.413 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.409+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:10.413 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.409+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:10.413 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.409+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:10.451 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.447+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:24:10.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.447+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:24:10.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.448+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:24:10.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.449+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:24:10.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.449+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:24:10.453 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.450+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:24:10.454 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.451+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:24:10.454 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.451+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:24:10.454 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.451+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:10.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 66044928 bytes 2026-02-20T21:24:10.732 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.728+0000 7fd67aee6640 -1 reset not still connected to 0x55daae4380d0 2026-02-20T21:24:10.732 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.728+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:24:10.732 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.728+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423ad0 2026-02-20T21:24:10.732 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.728+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:24:10.732 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.728+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:24:10.732 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.728+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:10.732 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.728+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:10.733 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.729+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:10.733 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.729+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:10.734 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.730+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:10.734 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.730+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:10.735 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.731+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f95f0 2026-02-20T21:24:10.735 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.731+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:24:10.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.732+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:10.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.732+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:10.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.732+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:10.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.732+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:10.743 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.740+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:10.986 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.982+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:24:10.988 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.984+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:24:10.990 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.986+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:10.990 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.986+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:10.990 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.986+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:10.990 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.986+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:10.990 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.986+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:10.990 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:10.986+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:10.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 94337024 bytes 2026-02-20T21:24:11.902 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:11.899+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:11.980 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:11.976+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:24:11.980 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:11.976+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:24:11.982 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:11.978+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:11.982 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:11.978+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:11.982 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:11.978+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:11.982 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:11.978+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:11.982 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:11.978+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:11.982 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:11.978+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:12.362 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.358+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:12.363 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.358+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:12.363 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.358+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:12.363 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.358+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:12.363 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.358+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:12.363 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.358+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:12.600 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.596+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:12.600 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.596+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:12.600 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.596+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:12.600 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.596+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:12.600 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.596+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:12.601 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.596+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:12.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 164225024 bytes 2026-02-20T21:24:12.675 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.671+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:12.676 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.671+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:12.676 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.671+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:12.676 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.671+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:12.676 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.672+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:12.676 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:12.672+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:13.091 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:13.087+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:24:13.093 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:13.089+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:24:13.093 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:13.089+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:13.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 250920960 bytes 2026-02-20T21:24:14.036 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.032+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:24:14.037 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.033+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:24:14.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 271555584 bytes 2026-02-20T21:24:14.385 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.382+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:24:14.386 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.382+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:24:14.386 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.382+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:24:14.387 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.384+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:24:14.604 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.600+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:24:14.604 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.600+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:24:14.604 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.600+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:24:14.604 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.600+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:24:14.606 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.602+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:24:14.606 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.602+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:24:14.606 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.602+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:24:14.606 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.602+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:24:14.673 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.670+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:14.675 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:14.672+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:15.077 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.073+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:24:15.077 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.073+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:15.078 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.074+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:15.078 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.074+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:15.078 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.074+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:15.079 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.074+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:15.079 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.074+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:15.131 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.127+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:24:15.131 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.127+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:24:15.132 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.128+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:15.133 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.129+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:15.133 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.129+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:15.134 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.130+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:15.134 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.130+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:15.135 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.130+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:15.329 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.325+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:15.329 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.325+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:15.329 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.325+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:15.330 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:15.327+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:15.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 342072320 bytes 2026-02-20T21:24:16.126 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:16.122+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:24:16.126 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:16.122+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:16.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 403223552 bytes 2026-02-20T21:24:17.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 429898752 bytes 2026-02-20T21:24:17.523 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:17.520+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:24:17.523 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:17.520+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:24:17.523 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:17.520+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:18.068 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:18.064+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:18.072 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:18.068+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:18.072 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:18.068+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d1e0 2026-02-20T21:24:18.072 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:18.068+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:24:18.072 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:18.068+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423ad0 2026-02-20T21:24:18.072 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:18.068+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:24:18.072 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:18.068+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:18.072 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:18.068+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:18.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 513752064 bytes 2026-02-20T21:24:18.501 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:18.497+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:24:18.501 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:18.497+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:18.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 558592000 bytes 2026-02-20T21:24:18.793 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:18.789+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:18.793 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:18.789+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:18.793 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:18.789+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:18.945 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:18.941+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e89751e0 2026-02-20T21:24:18.945 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:18.941+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:24:18.946 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:18.941+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:24:18.946 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:18.941+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:19.230 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:19.226+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:19.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:24:19.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:24:19.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:24:19.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:24:19.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:24:19.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_34 2026-02-20T21:24:19.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_34 journal 128M 2026-02-20T21:24:19.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:19.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:24:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:24:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:24:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_34 128M 2026-02-20T21:24:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:24:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:24:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:24:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:24:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_34 2026-02-20T21:24:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:24:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:24:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:24:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:24:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:24:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_34 2026-02-20T21:24:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:24:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:24:19.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-20T21:24:19.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:24:19.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_34 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:24:20.020 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:20.016+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:24:20.020 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:20.016+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:24:20.021 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:20.017+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 39717888 bytes 2026-02-20T21:24:20.373 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:20.369+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:24:20.373 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:20.369+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:24:20.373 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:20.369+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:24:20.374 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:20.369+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:24:20.374 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:20.369+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:24:20.375 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:20.372+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:20.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:20.419+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:24:20.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:20.419+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:20.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:20.419+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:20.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:20.419+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:20.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 74315776 bytes 2026-02-20T21:24:20.931 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:20.927+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:24:20.932 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:20.928+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:24:20.932 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:20.929+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:24:20.933 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:20.929+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 104448000 bytes 2026-02-20T21:24:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 117378048 bytes 2026-02-20T21:24:22.146 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:22.142+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:24:22.146 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:22.142+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:24:22.149 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:22.145+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:24:22.149 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:22.145+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:24:22.149 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:22.145+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:24:22.149 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:22.145+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:24:22.149 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:22.145+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:24:22.149 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:22.145+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:22.194 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:22.191+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:22.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:22.191+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:22.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:22.192+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:22.454 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:22.450+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:24:22.454 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:22.450+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:24:22.456 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:22.452+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:24:22.708 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:22.705+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:24:22.710 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:22.706+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:24:22.713 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:22.709+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:22.713 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:22.709+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:24:23.239 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:23.236+0000 7f413d8ae640 -1 reset not still connected to 0x55d26e401790 2026-02-20T21:24:23.240 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:23.236+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:24:23.240 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:23.236+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:24:23.240 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:23.236+0000 7f413d8ae640 -1 reset not still connected to 0x55d271027ee0 2026-02-20T21:24:23.240 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:23.236+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:24:23.240 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:23.236+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:23.240 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:23.236+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:23.240 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:23.236+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:23.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 191502336 bytes 2026-02-20T21:24:23.525 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:23.521+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:24:23.525 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:23.521+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:23.526 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:23.522+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:23.760 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:23.756+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc6f70 2026-02-20T21:24:23.761 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:23.757+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:24:23.761 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:23.757+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:24:23.761 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:23.757+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:24:23.815 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:23.811+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:23.816 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:23.812+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:23.816 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:23.812+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:23.816 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:23.812+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:24.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 237524992 bytes 2026-02-20T21:24:24.526 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:24.523+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:24.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 253919232 bytes 2026-02-20T21:24:24.783 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:24.780+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:24:24.783 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:24.780+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:24:24.783 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:24.780+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:24:24.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 276187136 bytes 2026-02-20T21:24:25.129 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:25.125+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:25.129 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:25.125+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:25.129 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:25.125+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:25.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 370006016 bytes 2026-02-20T21:24:25.911 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:25.907+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:26.044 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:26.040+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351fee0 2026-02-20T21:24:26.044 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:26.040+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:24:26.047 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:26.043+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:24:26.047 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:26.043+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:26.456 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.452+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d040 2026-02-20T21:24:26.456 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.452+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc920d0 2026-02-20T21:24:26.459 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.454+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:24:26.459 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.454+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:24:26.459 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.454+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:24:26.459 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.454+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:24:26.459 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.454+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:24:26.459 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.454+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:24:26.460 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.456+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:24:26.460 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.456+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:24:26.462 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.458+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:26.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:24:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:24:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:24:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:24:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:24:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_35 2026-02-20T21:24:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_35 journal 128M 2026-02-20T21:24:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:24:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:24:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:24:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_35 128M 2026-02-20T21:24:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:26.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:24:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:24:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:24:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:24:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_35 2026-02-20T21:24:26.735 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.732+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:26.735 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.732+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:26.735 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.732+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:26.781 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:26.777+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:24:26.781 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:26.778+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:24:26.783 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:26.780+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:24:26.785 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:26.781+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:26.785 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:26.782+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:24:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:24:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:26.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:24:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:24:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:24:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:24:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_35 2026-02-20T21:24:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:24:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:24:26.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-20T21:24:26.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:24:26.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_35 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:24:26.978 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.974+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423ad0 2026-02-20T21:24:26.979 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.975+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:26.979 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.975+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:26.980 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.976+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:26.980 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.976+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:26.980 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.976+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:26.980 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:26.976+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:27.086 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:27.082+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:27.087 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:27.083+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:27.088 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:27.084+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:27.088 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:27.084+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:27.088 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:27.084+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:27.371 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:27.367+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:24:27.371 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:27.367+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:24:27.586 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:27.582+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:27.670 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:27.665+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:24:27.670 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:27.665+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:24:27.670 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:27.665+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:27.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 72073216 bytes 2026-02-20T21:24:28.390 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.386+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f95f0 2026-02-20T21:24:28.390 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.386+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:24:28.390 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.386+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92f70 2026-02-20T21:24:28.390 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.386+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:28.390 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.386+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:28.391 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.386+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:28.391 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.386+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:28.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 107551744 bytes 2026-02-20T21:24:28.628 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.624+0000 7fd67aee6640 -1 reset not still connected to 0x55daae4380d0 2026-02-20T21:24:28.628 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.624+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:24:28.629 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.624+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423ad0 2026-02-20T21:24:28.630 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.626+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:24:28.630 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.627+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:28.632 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.628+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:28.632 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.628+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:28.632 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.628+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:28.632 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.628+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:28.632 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.628+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:28.878 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.874+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:28.878 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.874+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:28.878 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.874+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:28.878 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:28.874+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:28.936 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:28.932+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:29.109 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:29.106+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:24:29.110 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:29.106+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:24:29.110 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:29.106+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:29.111 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:29.107+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:29.112 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:29.109+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:29.113 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:29.109+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:29.412 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:29.408+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:29.413 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:29.410+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:29.413 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:29.410+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:29.413 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:29.410+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:29.413 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:29.410+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:24:29.413 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:29.410+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:29.427 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:29.423+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:29.450 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:29.447+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:24:29.450 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:29.447+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:24:29.451 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:29.448+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:29.868 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:29.864+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:24:29.868 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:29.864+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:29.869 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:29.866+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:29.871 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:29.867+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:29.934 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:29.930+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351fee0 2026-02-20T21:24:29.935 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:29.932+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e89751e0 2026-02-20T21:24:29.935 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:29.932+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:24:29.936 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:29.932+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:24:29.936 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:29.932+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:30.358 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:30.354+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:24:30.358 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:30.354+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:24:30.359 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:30.354+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:24:30.359 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:30.354+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:24:30.359 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:30.354+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:24:30.359 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:30.356+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:24:30.360 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:30.356+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:24:30.360 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:30.356+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:30.468 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:30.464+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:24:30.468 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:30.464+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:24:30.468 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:30.464+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:30.469 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:30.465+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:30.469 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:30.466+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:30.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 193306624 bytes 2026-02-20T21:24:30.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 215190528 bytes 2026-02-20T21:24:30.926 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:30.922+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:30.926 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:30.922+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:30.926 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:30.922+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:31.545 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:31.541+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:24:31.548 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:31.545+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:31.549 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:31.545+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:24:31.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 253477888 bytes 2026-02-20T21:24:31.765 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:31.762+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d520 2026-02-20T21:24:31.765 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:31.762+0000 7fd67aee6640 -1 reset not still connected to 0x55daae439ad0 2026-02-20T21:24:31.765 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:31.762+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:24:31.768 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:31.764+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:24:31.768 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:31.764+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:24:31.769 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:31.764+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:24:32.157 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:24:32.155+0000 7f03c8435640 -1 reset not still connected to 0x55af9f5496c0 2026-02-20T21:24:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 293873664 bytes 2026-02-20T21:24:32.398 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:32.394+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:24:32.399 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:32.394+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:24:32.399 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:32.394+0000 7f413d8ae640 -1 reset not still connected to 0x55d271027ee0 2026-02-20T21:24:32.399 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:32.394+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:24:32.399 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:32.394+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:32.399 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:32.394+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:32.399 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:32.394+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:32.399 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:32.394+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:32.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 323977216 bytes 2026-02-20T21:24:33.379 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:33.376+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:24:33.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 369315840 bytes 2026-02-20T21:24:33.825 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:33.822+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:33.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:24:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:24:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:24:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:24:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:24:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_36 2026-02-20T21:24:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_36 journal 128M 2026-02-20T21:24:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:33.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:24:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:24:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:24:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_36 128M 2026-02-20T21:24:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:24:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:33.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:24:33.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:24:33.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:24:33.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:33.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_36 2026-02-20T21:24:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:24:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:24:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:24:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:24:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:24:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_36 2026-02-20T21:24:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:33.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:24:33.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:24:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=9 2026-02-20T21:24:33.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:24:33.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_36 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:24:34.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 46384128 bytes 2026-02-20T21:24:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 88240128 bytes 2026-02-20T21:24:35.132 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.128+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:24:35.132 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.128+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:24:35.132 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.128+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:24:35.132 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.128+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:24:35.132 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.128+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:24:35.135 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.131+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:24:35.135 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.131+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:24:35.135 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.131+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:24:35.135 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.131+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:35.257 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:35.253+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:24:35.258 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:35.255+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:35.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.419+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d040 2026-02-20T21:24:35.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.419+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc920d0 2026-02-20T21:24:35.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.419+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:24:35.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.419+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:24:35.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.419+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:24:35.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.419+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:24:35.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.419+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:24:35.424 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.419+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:24:35.425 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.422+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:24:35.425 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.422+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:24:35.426 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.422+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:24:35.426 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:35.422+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:35.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 126156800 bytes 2026-02-20T21:24:37.342 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:37.338+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:24:37.343 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:37.339+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:24:37.343 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:37.339+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:37.343 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:37.340+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:37.343 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:37.340+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:37.344 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:37.340+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:37.643 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:37.639+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:24:37.643 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:37.639+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:24:37.643 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:37.639+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:24:37.644 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:37.640+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:24:37.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 180387840 bytes 2026-02-20T21:24:37.878 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:37.875+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:24:37.878 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:37.875+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:24:37.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 185869312 bytes 2026-02-20T21:24:38.309 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.305+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:24:38.310 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.307+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:24:38.312 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.308+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:24:38.312 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.308+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:24:38.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 218940416 bytes 2026-02-20T21:24:38.705 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.702+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:24:38.708 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.704+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:24:38.708 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.704+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:24:38.708 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.704+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:24:38.708 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.704+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:24:38.708 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.704+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:24:38.708 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.704+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:24:38.708 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.704+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:24:38.708 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.704+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:38.708 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:38.704+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:24:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 234452992 bytes 2026-02-20T21:24:39.231 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.227+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:24:39.231 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.227+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:24:39.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.242+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:39.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.243+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:24:39.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.244+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:39.248 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.244+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:24:39.248 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.244+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:39.248 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.244+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:39.610 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.606+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:24:39.610 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.606+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:24:39.610 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.606+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:39.611 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.608+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:24:39.612 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.608+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:39.613 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:39.610+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:40.026 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.022+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:24:40.027 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.022+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:24:40.028 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.024+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:24:40.124 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.120+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:40.124 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.120+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:40.126 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.123+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:40.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 327464960 bytes 2026-02-20T21:24:40.292 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.288+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:24:40.292 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.288+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:24:40.292 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.289+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:24:40.294 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.290+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:40.294 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.290+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:40.294 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.290+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:40.294 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.290+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:40.350 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.346+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:40.350 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.346+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:40.350 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.346+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:40.350 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.346+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:40.351 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.348+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:24:40.352 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.348+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:40.370 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:40.366+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:24:40.373 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:40.369+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:24:40.373 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:40.369+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:24:40.503 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.499+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:40.504 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.500+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:40.505 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.501+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:40.718 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.714+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:40.719 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.716+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:40.719 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.716+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:40.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.717+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:40.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.717+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:40.721 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.717+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:24:40.721 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.717+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:40.789 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.786+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:24:40.791 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:40.787+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:24:40.875 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.871+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:24:40.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.871+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:24:40.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.871+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:40.876 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.871+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:40.877 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.873+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:40.877 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.873+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:40.877 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.873+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:40.877 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.873+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:24:40.878 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:40.873+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:41.113 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:41.109+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:24:41.113 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:41.109+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:24:41.113 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:41.109+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:41.150 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:41.146+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:41.151 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:41.147+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:24:41.151 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:41.147+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:41.151 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:41.147+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:24:41.151 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:41.147+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:41.152 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:41.149+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 374957056 bytes 2026-02-20T21:24:41.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 409099264 bytes 2026-02-20T21:24:42.207 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:42.203+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:42.207 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:42.203+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:42.208 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:42.205+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:42.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 434163712 bytes 2026-02-20T21:24:42.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 474536960 bytes 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_37 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_37 journal 128M 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:24:42.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_37 128M 2026-02-20T21:24:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:24:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:24:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:24:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:24:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:42.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_37 2026-02-20T21:24:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:24:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:24:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:24:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:24:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:24:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_37 2026-02-20T21:24:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:24:42.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:24:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-20T21:24:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:24:42.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_37 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:24:44.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 62782464 bytes 2026-02-20T21:24:44.074 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:44.071+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:24:44.074 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:44.071+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:24:44.074 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:44.071+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:24:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 69090304 bytes 2026-02-20T21:24:44.517 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:44.513+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:44.517 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:44.513+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:44.518 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:44.513+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:44.518 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:44.513+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:24:44.518 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:44.513+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 92364800 bytes 2026-02-20T21:24:44.803 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:44.797+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:44.803 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:44.800+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:44.804 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:44.800+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:44.804 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:44.800+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:44.804 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:44.801+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:24:44.804 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:44.801+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:45.009 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:45.006+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:24:45.010 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:45.007+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:24:45.010 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:45.007+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:24:45.010 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:45.007+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:24:45.011 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:45.007+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:24:45.011 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:45.007+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:24:45.011 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:45.007+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:24:45.011 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:45.007+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:24:45.013 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:45.009+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:45.013 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:45.009+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:24:45.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 121925632 bytes 2026-02-20T21:24:45.339 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:45.335+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:24:45.339 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:45.335+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:24:46.123 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:46.119+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:24:46.123 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:46.119+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:24:46.123 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:46.119+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:24:46.124 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:46.121+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:24:46.124 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:46.121+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:46.125 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:46.121+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:24:46.178 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:46.175+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:46.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 198505472 bytes 2026-02-20T21:24:46.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 199117824 bytes 2026-02-20T21:24:46.624 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:46.620+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:24:46.624 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:46.620+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:46.624 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:46.620+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:46.798 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:46.794+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9110 2026-02-20T21:24:46.798 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:46.794+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:24:46.798 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:46.794+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:24:47.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 261348352 bytes 2026-02-20T21:24:48.478 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:48.474+0000 7fd67aee6640 -1 reset not still connected to 0x55daae439ad0 2026-02-20T21:24:48.478 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:48.474+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:24:48.479 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:48.475+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:24:48.479 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:48.475+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:24:48.479 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:48.475+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:24:48.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 320313344 bytes 2026-02-20T21:24:48.867 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:48.863+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:24:48.867 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:48.863+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:24:48.870 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:48.866+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:24:48.870 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:48.866+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:24:48.870 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:48.866+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:24:48.925 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:48.921+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:24:48.927 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:48.924+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:48.930 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:48.926+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:24:48.930 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:48.926+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:48.930 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:48.926+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:24:48.930 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:48.926+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:48.930 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:48.926+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 342687744 bytes 2026-02-20T21:24:49.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 353732608 bytes 2026-02-20T21:24:49.185 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:49.182+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:24:49.187 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:49.183+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:49.187 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:49.183+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:24:49.187 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:49.183+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:49.187 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:49.183+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:24:49.187 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:49.183+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:49.188 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:49.183+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:49.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 354342912 bytes 2026-02-20T21:24:49.535 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:49.532+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:24:49.535 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:49.532+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:24:49.536 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:49.533+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:24:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:24:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:24:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:24:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:24:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:24:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_38 2026-02-20T21:24:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_38 journal 128M 2026-02-20T21:24:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:24:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:24:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:24:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:49.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_38 128M 2026-02-20T21:24:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:24:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:24:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:24:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:24:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:49.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_38 2026-02-20T21:24:49.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:24:49.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:49.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:24:49.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:24:49.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:24:49.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:24:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_38 2026-02-20T21:24:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:24:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:24:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-20T21:24:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:24:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_38 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:24:50.143 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:50.140+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:50.144 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:50.140+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:50.144 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:50.141+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:50.346 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:50.342+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:24:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 23896064 bytes 2026-02-20T21:24:50.451 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:50.447+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:24:50.451 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:50.447+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:24:50.451 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:50.448+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:24:50.452 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:50.448+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:50.455 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:50.451+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:24:50.455 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:50.451+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:24:50.455 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:50.451+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:24:51.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 63446016 bytes 2026-02-20T21:24:51.676 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.672+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:24:51.677 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.673+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:24:51.677 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.674+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:24:51.677 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.674+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:24:51.677 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.674+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:24:51.736 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.733+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:24:51.736 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.733+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:24:51.737 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.734+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:24:51.738 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.734+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:24:51.738 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.734+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:24:51.738 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.734+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:24:51.738 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.734+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:24:51.738 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:51.735+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:24:51.866 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:51.863+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:51.868 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:51.864+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:24:51.868 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:51.864+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:51.868 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:51.864+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:24:51.868 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:51.864+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 128286720 bytes 2026-02-20T21:24:52.278 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:52.274+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:24:52.634 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:52.630+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:52.699 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:52.695+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:24:53.674 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:53.670+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:24:53.676 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:53.672+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:24:53.676 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:53.672+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:24:53.676 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:53.672+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:53.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 176556032 bytes 2026-02-20T21:24:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 177188864 bytes 2026-02-20T21:24:54.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 231149568 bytes 2026-02-20T21:24:55.589 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:55.586+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:24:56.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 323770368 bytes 2026-02-20T21:24:56.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 327898112 bytes 2026-02-20T21:24:56.348 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:56.344+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:24:56.349 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:56.345+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:24:56.349 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:56.345+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:24:56.349 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:56.345+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:24:56.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 364305408 bytes 2026-02-20T21:24:56.834 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:56.830+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:24:56.972 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-20T21:24:56.973 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-20T21:24:56.973 INFO:tasks.rbd_mirror_thrash:waiting for 4.409275403808797 secs before reviving daemons 2026-02-20T21:24:56.974 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:24:56.971+0000 7f2c062fc640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 91402) UID: 1000 2026-02-20T21:24:56.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:24:56.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:24:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:24:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:24:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:24:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_39 2026-02-20T21:24:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_39 journal 128M 2026-02-20T21:24:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:24:56.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:24:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:24:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_39 128M 2026-02-20T21:24:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:24:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:24:56.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:24:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:24:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:24:56.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_39 2026-02-20T21:24:57.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:24:57.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:57.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:24:57.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:24:57.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:24:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:24:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_39 2026-02-20T21:24:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:24:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:24:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:24:57.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:24:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=5 2026-02-20T21:24:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:24:57.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_39 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:24:57.237 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.233+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:24:57.239 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.235+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92f70 2026-02-20T21:24:57.239 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.235+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:24:57.239 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.235+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:24:57.239 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.235+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:24:57.239 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.235+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:24:57.239 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.235+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:24:57.239 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.235+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:24:57.936 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.931+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:24:57.936 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.931+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:24:57.937 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.933+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:57.937 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.933+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:57.937 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.933+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:57.937 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.933+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:57.938 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.934+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:57.938 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.934+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:24:57.938 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:57.934+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:58.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 59478016 bytes 2026-02-20T21:24:58.309 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.306+0000 7fc9d03ed640 -1 reset not still connected to 0x5576df3755f0 2026-02-20T21:24:58.311 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.308+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:24:58.314 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.310+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:24:58.314 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.310+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:24:58.314 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.310+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:24:58.314 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.310+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:24:58.314 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.310+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:24:58.315 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.310+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:24:58.315 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.311+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:24:58.315 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.311+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:58.315 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.311+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:24:58.511 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.506+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:24:58.511 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.506+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:24:58.511 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.506+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:24:58.511 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.506+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:24:58.786 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.782+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:24:58.787 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.784+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:24:58.788 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.784+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:24:58.789 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.785+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:24:58.789 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.785+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:24:58.789 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.785+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:24:58.789 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.785+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:24:58.790 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.785+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:24:58.790 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.785+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:24:58.790 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.785+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:58.790 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:58.785+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:24:58.964 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.959+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:24:58.964 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.959+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:24:58.964 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.960+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:24:58.964 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.960+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:24:58.965 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.960+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:24:58.965 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.960+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:24:58.965 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.960+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:24:58.965 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.961+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:24:58.965 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:24:58.961+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:24:59.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 134924288 bytes 2026-02-20T21:24:59.316 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:59.312+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:24:59.316 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:59.312+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:24:59.316 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:24:59.312+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:24:59.807 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:59.803+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:24:59.807 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:24:59.803+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:25:00.064 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:00.060+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:25:00.064 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:00.060+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:25:00.065 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:00.061+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:25:00.065 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:00.061+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:25:00.065 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:00.061+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:25:00.066 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:00.062+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:25:00.066 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:00.062+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:25:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 173247488 bytes 2026-02-20T21:25:01.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 245128192 bytes 2026-02-20T21:25:01.383 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-20T21:25:01.383 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:25:01.383 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-20T21:25:01.383 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T21:25:01.383 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T21:25:01.386 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T21:25:01.386 INFO:tasks.rbd_mirror_thrash:waiting for 107.87142586618575 secs before thrashing 2026-02-20T21:25:01.735 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:01.731+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:01.738 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:01.734+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:02.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:25:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:25:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:25:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:25:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:25:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_40 2026-02-20T21:25:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_40 journal 128M 2026-02-20T21:25:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:25:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:25:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:25:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_40 128M 2026-02-20T21:25:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:25:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:25:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:25:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:25:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_40 2026-02-20T21:25:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:25:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:25:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:25:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:25:02.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:25:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_40 2026-02-20T21:25:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:25:02.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:25:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-20T21:25:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:25:02.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_40 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:25:02.747 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:02.743+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:25:02.747 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:02.743+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:02.747 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:02.743+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:02.748 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:02.744+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:02.748 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:02.744+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:02.748 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:02.744+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:03.214 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.211+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:25:03.215 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.211+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:25:03.215 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.211+0000 7f413d8ae640 -1 reset not still connected to 0x55d271027ee0 2026-02-20T21:25:03.215 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.211+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:25:03.215 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.211+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:25:03.216 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.211+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:25:03.216 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.212+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:25:03.216 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.212+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:25:03.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 57913344 bytes 2026-02-20T21:25:03.400 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.396+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:25:03.400 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.396+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:25:03.400 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.396+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:25:03.401 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.397+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:25:03.401 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.397+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:25:03.401 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:03.398+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:25:03.920 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:03.916+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:25:03.921 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:03.917+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:03.921 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:03.917+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:03.921 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:03.917+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:03.921 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:03.917+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:03.921 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:03.917+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 103680000 bytes 2026-02-20T21:25:04.180 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:04.176+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:04.180 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:04.176+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:04.180 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:04.176+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:04.181 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:04.176+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:04.181 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:04.176+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:04.181 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:04.177+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:05.061 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-20T21:25:05.058+0000 7fdf55ce9640 -1 bdev(0x55a614e8f400 /var/lib/ceph/osd/cluster2-2/block) aio_submit retries 1 2026-02-20T21:25:05.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 141501440 bytes 2026-02-20T21:25:05.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 171746304 bytes 2026-02-20T21:25:06.262 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:06.258+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:06.262 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:06.258+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:06.262 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:06.258+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:06.294 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:06.290+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:25:06.294 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:06.290+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:25:06.295 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:06.290+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:25:06.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 213769216 bytes 2026-02-20T21:25:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 214389760 bytes 2026-02-20T21:25:06.946 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:06.942+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:25:06.946 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:06.942+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:06.946 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:06.942+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:06.947 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:06.942+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:06.947 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:06.942+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:06.947 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:06.942+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:07.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 266101760 bytes 2026-02-20T21:25:07.646 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:07.642+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:07.647 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:07.642+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:07.647 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:07.642+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:07.648 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:07.644+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:07.662 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:07.658+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:07.708 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:07.704+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:25:07.708 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:07.704+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:25:07.709 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:07.705+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:07.709 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:07.705+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:07.712 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:07.709+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:07.713 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:07.709+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:07.713 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:07.709+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 278932480 bytes 2026-02-20T21:25:08.343 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:08.339+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:25:08.343 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:08.339+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:25:08.343 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:08.339+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:25:08.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 303601664 bytes 2026-02-20T21:25:08.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 307918848 bytes 2026-02-20T21:25:09.086 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:09.082+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:25:09.089 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:09.085+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92f70 2026-02-20T21:25:09.089 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:09.085+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:25:09.089 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:09.085+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:09.089 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:09.085+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:09.089 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:09.085+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:09.089 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:09.085+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:09.089 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:09.085+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:09.089 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:09.085+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:09.528 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.524+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:09.529 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.524+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:09.529 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.525+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f6d00 2026-02-20T21:25:09.529 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.525+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351fee0 2026-02-20T21:25:09.531 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.527+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:25:09.531 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.527+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:25:09.531 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.527+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:25:09.660 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.656+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:25:09.660 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.656+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:25:09.661 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.656+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:25:09.661 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.656+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:09.661 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.656+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:25:09.661 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.657+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:25:09.662 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.658+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:25:09.662 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.658+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:09.662 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:09.658+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:09.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 359804928 bytes 2026-02-20T21:25:09.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 381888512 bytes 2026-02-20T21:25:10.077 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:10.073+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f6d00 2026-02-20T21:25:10.077 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:10.073+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351fee0 2026-02-20T21:25:10.078 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:10.074+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:25:10.078 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:10.074+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:25:10.079 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:10.074+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:25:10.658 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:25:10.655+0000 7f169e8b2640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x555d92aa9440 finish: resending after timeout 2026-02-20T21:25:10.903 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:10.899+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:25:10.903 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:10.899+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:25:10.903 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:10.900+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:25:10.903 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:10.900+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:25:10.903 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:10.900+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:10.904 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:10.900+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:10.904 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:10.900+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:11.281 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:11.278+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:25:11.283 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:11.278+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:25:11.285 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:11.281+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:11.285 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:11.281+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:11.286 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:11.283+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:11.287 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:11.283+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:11.287 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:11.283+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:11.287 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:11.283+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 450344960 bytes 2026-02-20T21:25:11.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 496706560 bytes 2026-02-20T21:25:11.971 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:11.967+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:11.972 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:11.967+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:11.973 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:11.969+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:11.973 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:11.969+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:11.973 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:11.969+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:12.337 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.333+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:25:12.337 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.333+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:12.337 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.333+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:25:12.338 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.333+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:12.338 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.333+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:12.338 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.334+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:12.338 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.334+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:12.386 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:12.382+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:12.386 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:12.382+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:12.386 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:12.382+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:12.386 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:12.382+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:12.387 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:12.384+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:12.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 516017152 bytes 2026-02-20T21:25:12.592 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.588+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:25:12.592 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.588+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:25:12.593 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.590+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:12.593 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.590+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:12.595 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.592+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:12.596 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.592+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:12.596 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.592+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:12.596 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:12.592+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:12.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 533296128 bytes 2026-02-20T21:25:13.043 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:13.039+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:25:13.044 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:13.039+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:25:13.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 555044864 bytes 2026-02-20T21:25:13.654 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:13.651+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:25:13.655 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:13.651+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:25:13.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 614490112 bytes 2026-02-20T21:25:13.835 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:13.831+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:25:13.835 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:13.831+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:14.063 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:14.059+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:25:14.064 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:14.060+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:25:14.064 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:14.060+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:25:14.064 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:14.060+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:25:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 648300544 bytes 2026-02-20T21:25:14.338 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:14.334+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:25:14.341 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:14.337+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:14.341 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:14.337+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:25:14.600 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:14.596+0000 7fd67aee6640 -1 reset not still connected to 0x55daae439ad0 2026-02-20T21:25:14.600 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:14.597+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:25:14.601 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:14.598+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:25:14.602 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:14.598+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:14.602 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:14.599+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:25:14.781 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:14.777+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f14cea0 2026-02-20T21:25:14.782 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:14.777+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:25:14.782 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:14.779+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:25:14.783 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:14.779+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:25:14.783 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:14.779+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:25:14.784 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:14.780+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:25:14.784 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:14.780+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:25:15.008 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:15.004+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:16.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:25:16.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:25:16.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:25:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:25:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:25:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_41 2026-02-20T21:25:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_41 journal 128M 2026-02-20T21:25:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:25:16.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:25:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:25:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_41 128M 2026-02-20T21:25:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:25:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:25:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:25:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:25:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_41 2026-02-20T21:25:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:25:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:25:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:25:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:25:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:25:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_41 2026-02-20T21:25:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:25:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:25:16.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-20T21:25:16.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:25:16.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_41 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:25:16.191 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:16.187+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:25:16.193 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:16.189+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:16.193 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:16.189+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:16.193 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:16.189+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:16.193 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:16.189+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:16.193 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:16.189+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:16.636 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.633+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:16.659 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.655+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:25:16.659 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.655+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:16.662 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.659+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:25:16.662 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.659+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:16.662 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.659+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:16.662 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.659+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:16.662 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.659+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 33415168 bytes 2026-02-20T21:25:16.876 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:25:16.873+0000 7f169e8b2640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x555d92aa9440 finish: resending after timeout 2026-02-20T21:25:17.003 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.999+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:25:17.003 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.999+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:25:17.003 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.999+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:25:17.003 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.999+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:17.003 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.999+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:17.003 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:16.999+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 63147008 bytes 2026-02-20T21:25:17.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 73370624 bytes 2026-02-20T21:25:17.575 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:17.572+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:25:17.576 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:17.572+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:18.063 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:18.059+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:25:18.063 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:18.059+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:25:18.411 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:18.408+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:25:18.411 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:18.408+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:25:18.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 144782336 bytes 2026-02-20T21:25:19.720 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:19.716+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:25:19.723 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:19.720+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:19.723 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:19.720+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:20.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 224967680 bytes 2026-02-20T21:25:20.558 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.555+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:25:20.558 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.555+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:20.558 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.555+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:20.559 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.556+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:20.559 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.556+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:20.560 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.557+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:20.561 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.557+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:20.727 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:20.723+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:25:20.729 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:20.725+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:25:20.730 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:20.726+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:20.730 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:20.727+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:20.862 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.858+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d040 2026-02-20T21:25:20.862 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.858+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc920d0 2026-02-20T21:25:20.862 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.858+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:25:20.862 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.858+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:25:20.864 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.861+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:25:20.864 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.861+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:20.865 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.861+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:25:20.865 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.861+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:25:20.867 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.863+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:25:20.868 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.864+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:20.868 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:20.864+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:20.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 247903232 bytes 2026-02-20T21:25:21.166 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.162+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:21.166 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.162+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:21.166 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.162+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:21.168 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.165+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:21.176 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.172+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:25:21.176 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.172+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:25:21.176 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.172+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:25:21.176 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.172+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:21.177 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.174+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:25:21.180 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.176+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:25:21.180 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.176+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:21.180 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.176+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:21.361 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.358+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d040 2026-02-20T21:25:21.361 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.358+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc920d0 2026-02-20T21:25:21.363 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.360+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:25:21.363 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.360+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:25:21.364 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.361+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:25:21.364 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.361+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:25:21.364 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.361+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:25:21.364 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.361+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:21.367 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.364+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:25:21.368 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.365+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:25:21.368 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.365+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:25:21.368 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.365+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:25:21.369 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.365+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:21.369 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:21.365+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:21.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 295684096 bytes 2026-02-20T21:25:22.854 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:22.850+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:23.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 400697344 bytes 2026-02-20T21:25:23.605 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:23.602+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:25:23.605 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:23.602+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:25:23.606 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:23.602+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:25:23.945 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:23.942+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:25:23.945 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:23.942+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:25:23.947 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:23.944+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:25:23.947 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:23.944+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:25:23.947 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:23.944+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:25:23.947 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:23.944+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:25:24.396 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:24.392+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:25:24.396 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:24.392+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:25:24.396 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:24.393+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:25:24.396 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:24.393+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:25:24.396 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:24.393+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:25:24.396 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:24.393+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:25:24.397 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:24.393+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:25:24.399 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:24.396+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:25:24.400 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:24.396+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:25:24.402 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:24.398+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:25:24.402 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:24.399+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:25:24.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 445345792 bytes 2026-02-20T21:25:24.487 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:24.484+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:24.487 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:24.484+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:24.488 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:24.485+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:24.489 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:24.486+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 484107264 bytes 2026-02-20T21:25:25.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 491591680 bytes 2026-02-20T21:25:25.583 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:25.579+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:25:25.722 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:25.719+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:25:25.987 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:25:25.985+0000 7f03c8435640 -1 reset not still connected to 0x55afce0ee750 2026-02-20T21:25:25.987 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:25:25.985+0000 7f03c8435640 -1 reset not still connected to 0x55afcef5c410 2026-02-20T21:25:26.193 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:26.190+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:26.195 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:26.191+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:25:26.196 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:26.193+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:26.198 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:26.194+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:26.483 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:26.480+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:25:26.483 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:26.480+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:26.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 578235392 bytes 2026-02-20T21:25:26.732 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:26.728+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:25:26.733 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:26.728+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:25:27.015 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.011+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:27.016 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.013+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:25:27.016 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.013+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:25:27.018 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.014+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:27.018 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.014+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:27.018 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.014+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:27.018 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.014+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:27.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 601644032 bytes 2026-02-20T21:25:27.029 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.025+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:27.309 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.305+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:27.311 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.307+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:25:27.315 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.311+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:25:27.315 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.311+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:27.315 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.311+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 627671040 bytes 2026-02-20T21:25:27.814 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.810+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:27.814 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.810+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:27.814 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.810+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:27.815 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.810+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:27.815 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.810+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:27.815 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:27.811+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:28.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:25:28.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:25:28.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:25:28.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:25:28.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:25:28.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_42 2026-02-20T21:25:28.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_42 journal 128M 2026-02-20T21:25:28.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:28.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:28.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:28.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:25:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:25:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:25:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_42 128M 2026-02-20T21:25:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:25:28.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:28.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:25:28.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:25:28.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:25:28.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:28.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_42 2026-02-20T21:25:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:25:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:25:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:25:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:25:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:25:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_42 2026-02-20T21:25:28.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:25:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:25:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=11 2026-02-20T21:25:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:25:28.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 11s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_42 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:25:28.288 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:28.284+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:25:28.288 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:28.284+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:25:28.289 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:28.285+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:25:28.290 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:28.286+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:25:28.290 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:28.287+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:25:28.303 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:28.299+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:25:28.303 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:28.299+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:28.303 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:28.299+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:25:28.306 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:28.303+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:28.307 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:28.303+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:28.791 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:28.784+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:25:28.791 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:28.784+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:25:28.791 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:28.784+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:25:28.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 40939520 bytes 2026-02-20T21:25:29.110 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:29.106+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:25:29.111 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:29.106+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:25:29.111 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:29.106+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:25:29.111 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:29.106+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:25:29.126 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:29.122+0000 7fd67aee6640 -1 reset not still connected to 0x55daae439ad0 2026-02-20T21:25:29.126 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:29.122+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:25:29.127 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:29.124+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:25:29.127 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:29.124+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:29.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 75301888 bytes 2026-02-20T21:25:29.952 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:29.948+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:25:29.952 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:29.948+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:25:29.954 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:29.950+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:25:29.954 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:29.950+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:25:30.010 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:30.006+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:25:30.011 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:30.008+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:25:30.014 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:30.010+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:25:30.014 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:30.010+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:25:30.014 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:30.010+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:25:30.072 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:30.068+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:25:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 113011712 bytes 2026-02-20T21:25:30.942 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:30.938+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:25:31.175 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.171+0000 7fd67aee6640 -1 reset not still connected to 0x55daae439ad0 2026-02-20T21:25:31.176 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.173+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:25:31.180 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.177+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:25:31.180 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.177+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:31.180 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.177+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:25:31.240 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:31.236+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:25:31.332 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.328+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:25:31.332 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.328+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:25:31.334 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.330+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:31.334 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.330+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:31.335 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.331+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:31.335 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.331+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:31.335 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.331+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:31.336 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.332+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:31.420 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.416+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92f70 2026-02-20T21:25:31.422 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.418+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:25:31.422 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.418+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:31.422 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.418+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:31.422 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.418+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:31.422 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.418+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:31.422 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.418+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:31.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.419+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:31.578 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:31.574+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:25:31.579 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:31.575+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:25:31.579 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:31.575+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:25:31.580 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:31.576+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:25:31.641 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.637+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d040 2026-02-20T21:25:31.641 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.637+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:25:31.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.639+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:25:31.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.639+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:25:31.643 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.639+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:31.643 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.639+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:25:31.644 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.641+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:31.645 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:31.641+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:31.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 185137152 bytes 2026-02-20T21:25:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 205450240 bytes 2026-02-20T21:25:32.356 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:32.352+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:32.356 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:32.352+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:32.357 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:32.354+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:32.357 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:32.354+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:32.358 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:32.354+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:32.359 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:32.356+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:32.618 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:32.614+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:25:32.618 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:32.614+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:32.620 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:32.616+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:25:32.622 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:32.618+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:32.622 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:32.618+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:32.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 257406976 bytes 2026-02-20T21:25:32.859 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:32.855+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:25:32.859 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:32.855+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:32.859 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:32.855+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:25:33.000 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:32.996+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:25:33.000 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:32.996+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:33.078 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:33.074+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:25:33.078 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:33.074+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:25:33.079 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:33.075+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:25:33.079 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:33.075+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:25:33.079 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:33.075+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:25:33.080 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:33.076+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:25:33.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 263730176 bytes 2026-02-20T21:25:33.676 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:33.672+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:25:33.676 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:33.672+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:25:33.834 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:33.830+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:25:33.834 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:33.830+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:25:33.835 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:33.831+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:33.835 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:33.831+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:33.835 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:33.831+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:33.835 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:33.831+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:33.835 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:33.831+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:33.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 313630720 bytes 2026-02-20T21:25:34.109 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:25:34.106+0000 7f03c8435640 -1 reset not still connected to 0x55afce0ee750 2026-02-20T21:25:34.110 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:25:34.106+0000 7f03c8435640 -1 reset not still connected to 0x55afcef5c410 2026-02-20T21:25:34.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 375764992 bytes 2026-02-20T21:25:34.747 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:34.743+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:25:34.747 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:34.744+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:34.747 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:34.744+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:34.748 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:34.744+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:35.594 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:35.590+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:35.594 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:35.590+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:35.594 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:35.590+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:35.594 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:35.590+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:35.594 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:35.590+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:35.594 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:35.590+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:35.622 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:35.618+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:25:35.623 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:35.620+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:35.624 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:35.620+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:25:35.626 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:35.622+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:35.626 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:35.622+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:36.047 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:36.044+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:36.049 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:36.045+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:25:36.049 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:36.045+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:25:36.051 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:36.047+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:36.051 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:36.047+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:36.053 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:36.049+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:36.053 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:36.049+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:36.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 457055232 bytes 2026-02-20T21:25:36.569 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:36.565+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:25:36.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 466168832 bytes 2026-02-20T21:25:36.620 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:36.617+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:36.621 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:36.617+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:36.622 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:36.618+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:36.622 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:36.618+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:36.623 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:36.619+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:36.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 485785600 bytes 2026-02-20T21:25:36.721 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:36.717+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:25:36.725 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:36.720+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:25:36.725 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:36.720+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:25:36.725 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:36.720+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:25:36.725 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:36.720+0000 7f413d8ae640 -1 reset not still connected to 0x55d283729e10 2026-02-20T21:25:37.241 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:37.237+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:25:37.241 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:37.237+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:25:37.248 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:37.244+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:37.249 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:37.245+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:37.249 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:37.245+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:37.249 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:37.245+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:37.249 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:37.245+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:37.249 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:37.245+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:25:37.249 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:37.245+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:25:37.251 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:37.247+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:37.252 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:37.249+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:37.252 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:37.249+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:37.268 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.264+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:37.274 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.271+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:25:37.275 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.271+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:37.277 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.273+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:25:37.278 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.274+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:25:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 521226240 bytes 2026-02-20T21:25:37.801 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.797+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:25:37.803 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.799+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:25:37.803 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.799+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:25:37.803 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.800+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:25:37.805 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.801+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:25:37.805 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.801+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:37.805 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.801+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:25:37.806 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.802+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:25:37.807 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.802+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:25:37.807 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.802+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:37.807 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.802+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:37.807 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.802+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:25:37.807 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.802+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:37.896 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:37.892+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:37.897 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:37.893+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:37.897 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:37.893+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:37.898 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:37.893+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:37.942 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:37.939+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:38.183 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:38.179+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:38.184 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:38.181+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 581138432 bytes 2026-02-20T21:25:38.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 581737472 bytes 2026-02-20T21:25:38.693 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:38.689+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:25:38.694 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:38.690+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:25:38.694 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:38.690+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:25:38.694 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:38.690+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:25:38.694 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:38.690+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:25:38.698 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:38.695+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e21464e0 2026-02-20T21:25:38.699 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:38.695+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:25:38.699 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:38.695+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:25:38.699 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:38.695+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:25:38.836 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:38.832+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:38.836 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:38.832+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 606108672 bytes 2026-02-20T21:25:39.018 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:39.014+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:25:39.021 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:39.017+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:25:39.115 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:39.110+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:25:39.115 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:39.111+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:25:39.152 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.148+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:25:39.154 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.150+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:25:39.154 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.150+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:39.154 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.150+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:25:39.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:25:39.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:25:39.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:25:39.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:25:39.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:25:39.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_43 2026-02-20T21:25:39.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_43 journal 128M 2026-02-20T21:25:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:25:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:25:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:25:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_43 128M 2026-02-20T21:25:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:25:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:25:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:25:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:25:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_43 2026-02-20T21:25:39.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.241+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d040 2026-02-20T21:25:39.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.241+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc920d0 2026-02-20T21:25:39.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.241+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:25:39.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.241+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:25:39.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.241+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:25:39.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.241+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:25:39.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.241+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:25:39.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.241+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:39.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.243+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:25:39.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.243+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:25:39.249 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.245+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:25:39.249 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.245+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:25:39.249 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.245+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:39.251 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.247+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:39.251 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.247+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f95f0 2026-02-20T21:25:39.251 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.247+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92340 2026-02-20T21:25:39.251 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.247+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:25:39.251 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.247+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:39.251 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.247+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:39.251 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.247+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:25:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:25:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:25:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:25:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:25:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_43 2026-02-20T21:25:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:25:39.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:25:39.254 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.250+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:39.254 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.250+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:39.254 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:39.250+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=6 2026-02-20T21:25:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:25:39.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 6s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_43 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:25:39.346 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:39.343+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:40.006 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:40.002+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351fee0 2026-02-20T21:25:40.006 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:40.002+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:25:40.006 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:40.002+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:25:40.006 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:40.002+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:25:40.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 40046592 bytes 2026-02-20T21:25:40.028 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:40.025+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:40.029 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:40.026+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:40.029 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:40.026+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:40.029 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:40.026+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:40.203 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:40.199+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:25:40.205 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:40.202+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:40.205 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:40.202+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:40.205 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:40.202+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:40.206 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:40.202+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:40.206 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:40.202+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:40.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 58344448 bytes 2026-02-20T21:25:40.396 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:40.393+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:40.788 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:40.784+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351fee0 2026-02-20T21:25:40.788 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:40.784+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:25:40.789 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:40.784+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:25:40.789 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:40.784+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:25:40.792 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:40.788+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:25:40.793 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:40.788+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:40.793 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:40.788+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:40.793 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:40.788+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:40.793 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:40.788+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:40.793 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:40.788+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:40.831 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:40.827+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351fee0 2026-02-20T21:25:40.831 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:40.827+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1c70 2026-02-20T21:25:40.831 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:40.827+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:25:40.832 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:40.828+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:25:40.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 105218048 bytes 2026-02-20T21:25:41.461 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:41.457+0000 7f413d8ae640 -1 reset not still connected to 0x55d2710271e0 2026-02-20T21:25:41.463 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:41.460+0000 7f413d8ae640 -1 reset not still connected to 0x55d271027ee0 2026-02-20T21:25:41.464 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:41.460+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dde10 2026-02-20T21:25:41.464 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:41.460+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fedd0 2026-02-20T21:25:41.464 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:41.460+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b2ea0 2026-02-20T21:25:41.464 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:41.460+0000 7f413d8ae640 -1 reset not still connected to 0x55d280825040 2026-02-20T21:25:41.552 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:41.548+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:41.552 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:41.548+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:41.552 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:41.549+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:41.553 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:41.549+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:41.553 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:41.549+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:41.805 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:41.801+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:25:41.807 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:41.803+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:25:41.808 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:41.803+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:25:41.808 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:41.803+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:25:41.808 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:41.803+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:41.811 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:41.807+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:25:41.811 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:41.807+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:25:41.811 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:41.807+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:25:41.811 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:41.807+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:41.811 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:41.807+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:42.081 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:42.077+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:25:42.084 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:42.080+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:25:42.084 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:42.080+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:25:42.084 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:42.080+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:25:42.084 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:42.080+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:42.085 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:42.081+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:25:42.085 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:42.081+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:25:42.086 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:42.083+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:25:42.087 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:42.083+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:42.087 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:42.083+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:42.272 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:25:42.268+0000 7f03c8435640 -1 reset not still connected to 0x55afb9b1a8f0 2026-02-20T21:25:42.272 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:25:42.269+0000 7f03c8435640 -1 reset not still connected to 0x55afc57e7c70 2026-02-20T21:25:42.272 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:25:42.269+0000 7f03c8435640 -1 reset not still connected to 0x55afce0ee750 2026-02-20T21:25:42.272 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:25:42.269+0000 7f03c8435640 -1 reset not still connected to 0x55afcef5c410 2026-02-20T21:25:42.272 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:25:42.269+0000 7f03c8435640 -1 reset not still connected to 0x55afd0b6b860 2026-02-20T21:25:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 173201408 bytes 2026-02-20T21:25:42.551 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:42.548+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:25:42.730 INFO:tasks.ceph.cluster2.osd.1.trial163.stderr:2026-02-20T21:25:42.727+0000 7fa52b210640 -1 reset not still connected to 0x556133dc4750 2026-02-20T21:25:42.730 INFO:tasks.ceph.cluster2.osd.1.trial163.stderr:2026-02-20T21:25:42.727+0000 7fa52b210640 -1 reset not still connected to 0x556133dc5c70 2026-02-20T21:25:42.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 220323840 bytes 2026-02-20T21:25:43.133 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:43.130+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:25:43.151 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:43.147+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:25:43.151 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:43.147+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:43.151 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:43.147+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:43.151 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:43.147+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:43.151 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:43.147+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:43.153 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:43.148+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:43.153 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:43.148+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:43.462 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:43.458+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:25:43.462 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:43.458+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:25:43.463 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:43.459+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:25:43.463 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:43.459+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:25:43.464 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:43.459+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:25:43.464 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:43.459+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:25:43.464 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:43.459+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:25:44.098 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:44.094+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:25:44.100 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:44.096+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:25:44.101 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:44.097+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:25:44.103 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:44.099+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:25:44.103 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:44.099+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:25:44.103 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:44.099+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:25:44.103 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:44.099+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:25:44.208 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:44.205+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9110 2026-02-20T21:25:44.210 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:44.207+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:25:44.210 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:44.207+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:25:44.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 309985280 bytes 2026-02-20T21:25:44.676 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.672+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d040 2026-02-20T21:25:44.677 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.673+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc920d0 2026-02-20T21:25:44.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.675+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:25:44.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.675+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:25:44.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.675+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:25:44.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.675+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441110 2026-02-20T21:25:44.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.675+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:25:44.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.675+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:44.680 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.675+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:25:44.680 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.675+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:25:44.680 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.675+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:44.680 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.675+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:44.846 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.842+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:44.847 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.842+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:44.847 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:44.842+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:45.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 362204160 bytes 2026-02-20T21:25:45.204 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:45.200+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:45.204 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:45.200+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:45.206 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:45.202+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:25:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:25:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:25:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:25:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:25:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_44 2026-02-20T21:25:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_44 journal 128M 2026-02-20T21:25:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:45.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_44 128M 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:45.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:25:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:25:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:25:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:25:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:45.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_44 2026-02-20T21:25:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:25:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:25:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:25:45.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:25:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:25:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_44 2026-02-20T21:25:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:25:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:25:45.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-20T21:25:45.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:25:45.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_44 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:25:45.511 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:45.507+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:45.511 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:45.507+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:45.512 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:45.508+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:45.512 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:45.508+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:45.512 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:45.508+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:45.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 32625664 bytes 2026-02-20T21:25:46.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 33265664 bytes 2026-02-20T21:25:46.282 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:46.278+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:25:46.282 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:46.278+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:25:46.283 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:46.279+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:25:46.284 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:46.280+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:25:46.284 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:46.280+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:46.284 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:46.280+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:25:46.285 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:46.280+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:25:46.324 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:46.320+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:46.325 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:46.321+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:25:46.326 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:46.322+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:25:46.327 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:46.323+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 87898112 bytes 2026-02-20T21:25:47.225 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.222+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:25:47.227 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.223+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:25:47.227 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.223+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:47.229 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.225+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:25:47.229 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.225+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:25:47.229 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.225+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:47.229 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.225+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:47.229 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.225+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:47.229 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.225+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:47.286 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.282+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:25:47.286 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.282+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:47.286 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.282+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:47.288 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.284+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:47.434 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.431+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:47.434 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.431+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:47.437 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.434+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:47.437 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.434+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:47.437 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.434+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:47.438 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.435+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:47.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 159215616 bytes 2026-02-20T21:25:47.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:47.675+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:48.104 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:48.101+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:25:48.105 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:48.101+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:48.106 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:48.102+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:25:48.110 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:48.106+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:48.111 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:48.108+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:48.220 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:48.217+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:25:48.223 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:48.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:25:48.223 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:48.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:25:48.223 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:48.220+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:48.223 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:48.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:25:48.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:48.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:25:48.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:48.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:48.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:48.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:48.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:48.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:48.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:48.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:48.724 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:48.720+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:25:48.724 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:48.721+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:25:48.724 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:48.721+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:48.724 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:48.721+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:48.726 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:48.722+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:48.726 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:48.722+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:48.726 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:48.722+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:48.754 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:48.751+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:25:49.289 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:49.286+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae60d0 2026-02-20T21:25:49.291 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:49.287+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5f1e0 2026-02-20T21:25:49.316 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:49.313+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:49.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 230169600 bytes 2026-02-20T21:25:49.742 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:49.738+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:25:49.742 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:49.738+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:25:49.787 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:49.783+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:25:49.787 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:49.783+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:25:49.788 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:49.784+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:25:49.792 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:49.789+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:25:49.820 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:49.816+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:49.820 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:49.816+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:25:49.821 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:49.817+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:49.821 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:49.817+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:50.006 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:50.002+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:25:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 288307200 bytes 2026-02-20T21:25:50.288 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.284+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:25:50.289 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.285+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:50.289 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.285+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:25:50.585 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.581+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:25:50.586 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.582+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:25:50.587 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.583+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:25:50.589 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.585+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:50.589 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.585+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:25:50.589 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.585+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:25:50.589 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.585+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:50.589 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.585+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:50.589 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.585+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:50.589 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:50.585+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:51.097 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.093+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ee0d0 2026-02-20T21:25:51.099 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.095+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b0d00 2026-02-20T21:25:51.099 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.095+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1520 2026-02-20T21:25:51.100 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.096+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aba00 2026-02-20T21:25:51.100 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.096+0000 7fd67aee6640 -1 reset not still connected to 0x55dad126ac30 2026-02-20T21:25:51.100 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.096+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:25:51.101 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.097+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a270 2026-02-20T21:25:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 372015104 bytes 2026-02-20T21:25:51.451 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.447+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:25:51.451 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.447+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:25:51.451 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.448+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:51.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.448+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:51.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.448+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:51.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.448+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:51.480 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.476+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f9ba0 2026-02-20T21:25:51.480 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.476+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:25:51.481 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:51.478+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:25:51.488 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:51.484+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:25:51.490 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:51.486+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3143450 2026-02-20T21:25:51.490 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:51.486+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:25:51.490 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:51.486+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:25:51.491 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:51.487+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:25:51.491 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:51.487+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:25:51.491 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:51.487+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:25:51.685 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.6889262555788024) > weight (0.25) 2026-02-20T21:25:51.685 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5833815236093187) > weight (0.25) 2026-02-20T21:25:51.685 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-20T21:25:51.686 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-20T21:25:51.686 INFO:tasks.rbd_mirror_thrash:waiting for 9.541155014246481 secs before reviving daemons 2026-02-20T21:25:51.686 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:25:51.684+0000 7fd527d7f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 91669) UID: 1000 2026-02-20T21:25:52.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 439114752 bytes 2026-02-20T21:25:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:25:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:25:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:25:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:25:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:25:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_45 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_45 journal 128M 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_45 128M 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:25:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:25:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:25:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:25:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_45 2026-02-20T21:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_45 2026-02-20T21:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:25:52.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:25:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-20T21:25:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:25:52.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_45 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:25:52.625 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:52.621+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:52.626 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:52.621+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:52.639 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:52.635+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:52.640 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:52.637+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:52.640 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:52.637+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:52.640 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:52.637+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 16164864 bytes 2026-02-20T21:25:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 24763392 bytes 2026-02-20T21:25:53.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 62804992 bytes 2026-02-20T21:25:53.418 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d26c56f930 2026-02-20T21:25:53.418 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d26e400f70 2026-02-20T21:25:53.418 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1724e0 2026-02-20T21:25:53.418 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:25:53.418 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:25:53.419 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:25:53.419 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:25:53.419 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:25:53.419 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:25:53.419 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.416+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:25:53.420 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.416+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:25:53.772 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.768+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:25:53.772 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.768+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:25:53.772 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.768+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:53.772 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.768+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:53.773 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.768+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:53.773 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.768+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:53.773 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:53.768+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:54.113 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:54.109+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:25:54.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:54.110+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:25:54.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:54.110+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:25:54.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:54.110+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:25:54.115 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:54.110+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:25:54.115 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:54.110+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:25:54.299 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:54.295+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:54.299 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:54.295+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:54.299 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:54.295+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:54.299 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:54.295+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:54.635 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:54.630+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:25:54.638 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:54.634+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:25:54.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:54.634+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:25:54.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:54.634+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:54.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:54.634+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:25:54.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:54.634+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:25:54.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:54.634+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:54.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:54.634+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:54.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:54.634+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:54.639 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:54.634+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:54.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 135644160 bytes 2026-02-20T21:25:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 143328256 bytes 2026-02-20T21:25:55.545 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:55.541+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423ad0 2026-02-20T21:25:55.545 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:55.541+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:25:55.545 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:55.541+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:55.545 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:55.541+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:25:55.545 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:55.541+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:55.545 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:55.541+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:55.545 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:55.541+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:55.546 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:55.541+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:55.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 199995392 bytes 2026-02-20T21:25:56.268 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.264+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423ad0 2026-02-20T21:25:56.268 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.264+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:25:56.270 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.266+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:25:56.270 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.266+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:25:56.271 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.266+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:56.271 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.266+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:25:56.271 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.266+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:56.271 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.267+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:56.442 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.438+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:25:56.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.440+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:25:56.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.441+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:25:56.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.441+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:25:56.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.441+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:25:56.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.441+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:25:56.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.441+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:25:56.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.441+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:25:56.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.441+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:25:56.446 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.441+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:25:56.463 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:56.460+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:25:56.464 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:56.460+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:25:56.464 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:56.460+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:56.465 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:56.462+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:56.465 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:56.462+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:56.466 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:56.462+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:56.466 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:56.462+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 246775808 bytes 2026-02-20T21:25:56.804 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.800+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d040 2026-02-20T21:25:56.804 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.800+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc920d0 2026-02-20T21:25:56.804 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.800+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:25:56.804 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.800+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:25:56.805 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.800+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:25:56.805 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.801+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:25:56.805 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.801+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:25:56.805 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.801+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:25:56.806 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.801+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:25:56.806 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:25:56.801+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:25:57.048 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.044+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:25:57.052 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.048+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:25:57.054 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.050+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:25:57.054 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.050+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:25:57.054 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.050+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1930 2026-02-20T21:25:57.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 275996672 bytes 2026-02-20T21:25:57.204 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:57.200+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:25:57.204 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:57.200+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:25:57.204 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:57.200+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:25:57.205 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:57.200+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:25:57.205 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:57.200+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:25:57.205 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:57.200+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:25:57.741 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.737+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:25:57.742 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.738+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:25:57.742 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.738+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:25:57.743 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.739+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:25:57.743 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.739+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:25:57.743 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.739+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:25:57.745 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.741+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:57.745 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.741+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:25:57.745 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.741+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:25:57.745 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.741+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:25:57.746 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.741+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:25:57.746 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.742+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:57.746 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:57.742+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:57.753 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:57.749+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:25:57.757 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:57.753+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:25:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 357330944 bytes 2026-02-20T21:25:58.300 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:58.297+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:25:58.303 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:58.300+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:25:58.305 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:58.302+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:25:58.305 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:58.302+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:58.305 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:58.302+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:58.333 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:58.329+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:25:58.333 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:58.330+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:58.333 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:58.330+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:25:58.333 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:58.330+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:25:58.334 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:58.331+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:58.334 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:58.331+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:58.342 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:58.338+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:25:58.342 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:58.338+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:25:58.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 374714368 bytes 2026-02-20T21:25:58.564 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-20T21:25:58.561+0000 7fdf668cb640 -1 reset not still connected to 0x55a6521f9a00 2026-02-20T21:25:58.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 404738048 bytes 2026-02-20T21:25:59.338 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:59.334+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:25:59.339 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:59.334+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:25:59.339 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:59.335+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:25:59.342 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:59.338+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:25:59.342 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:25:59.338+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:25:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:25:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:25:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:25:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:25:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:25:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_46 2026-02-20T21:25:59.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_46 journal 128M 2026-02-20T21:25:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:25:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:25:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:25:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_46 128M 2026-02-20T21:25:59.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:25:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:25:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:25:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:25:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:25:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_46 2026-02-20T21:25:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:25:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:25:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:25:59.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:25:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:25:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_46 2026-02-20T21:25:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:25:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:25:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:25:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:25:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-20T21:25:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:25:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_46 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:25:59.723 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:59.719+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:25:59.724 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:59.719+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:25:59.724 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:25:59.719+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:00.064 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:00.060+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:26:00.065 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:00.061+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:26:00.065 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:00.061+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:26:00.065 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:00.061+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:00.066 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:00.062+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb2270 2026-02-20T21:26:00.066 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:00.062+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fc9ad0 2026-02-20T21:26:00.067 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:00.063+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:00.067 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:00.063+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa0b60 2026-02-20T21:26:00.068 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:00.064+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:00.614 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:00.608+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:00.614 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:00.609+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:00.614 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:00.609+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:00.614 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:00.609+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:00.615 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:00.609+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d1e0 2026-02-20T21:26:00.615 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:00.609+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:26:00.615 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:00.609+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:26:00.615 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:00.609+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:26:00.616 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:00.611+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:00.616 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:00.611+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:00.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 60339200 bytes 2026-02-20T21:26:00.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 63118336 bytes 2026-02-20T21:26:00.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 75882496 bytes 2026-02-20T21:26:01.227 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-20T21:26:01.227 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:26:01.228 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-20T21:26:01.228 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-20T21:26:01.228 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-20T21:26:01.231 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-20T21:26:01.231 INFO:tasks.rbd_mirror_thrash:waiting for 115.01285411594095 secs before thrashing 2026-02-20T21:26:01.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 161891328 bytes 2026-02-20T21:26:01.799 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:01.795+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d040 2026-02-20T21:26:01.799 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:01.796+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc920d0 2026-02-20T21:26:01.800 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:01.797+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc92820 2026-02-20T21:26:01.800 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:01.797+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:26:01.803 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:01.799+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:26:01.803 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:01.799+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fd0b60 2026-02-20T21:26:01.803 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:01.799+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:26:01.803 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:01.799+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:01.803 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:01.799+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:01.803 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:01.799+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:01.803 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:01.799+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:03.397 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:03.393+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:26:03.399 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:03.395+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:26:03.400 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:03.395+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:03.400 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:03.395+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:03.400 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:03.395+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:26:03.400 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:03.396+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:03.400 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:03.396+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:03.400 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:03.396+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:03.400 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:03.396+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:03.400 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:03.396+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 226195456 bytes 2026-02-20T21:26:03.904 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:03.901+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:03.905 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:03.901+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:03.905 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:03.901+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:03.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 243574784 bytes 2026-02-20T21:26:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 275819520 bytes 2026-02-20T21:26:04.466 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:04.463+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:04.466 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:04.463+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:04.467 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:04.463+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:05.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:05.243+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:26:05.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:05.243+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:05.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:05.243+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:05.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:05.244+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:05.249 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:05.245+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:05.520 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-20T21:26:05.516+0000 7fdf668cb640 -1 reset not still connected to 0x55a6513055f0 2026-02-20T21:26:05.641 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:05.637+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:05.641 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:05.637+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:05.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:05.637+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:05.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:05.637+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:05.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:05.637+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:05.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:05.637+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:05.642 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:05.637+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 386882560 bytes 2026-02-20T21:26:06.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 409664512 bytes 2026-02-20T21:26:06.050 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:06.046+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:06.112 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:06.107+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:06.112 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:06.107+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:06.113 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:06.107+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:06.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 419249152 bytes 2026-02-20T21:26:06.406 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:06.402+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:06.406 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:06.402+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:07.035 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:07.030+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:07.035 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:07.031+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:07.035 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:07.031+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:07.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 457290752 bytes 2026-02-20T21:26:07.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:26:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:26:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:26:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:26:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:26:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_47 2026-02-20T21:26:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_47 journal 128M 2026-02-20T21:26:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:07.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:26:07.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:07.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:26:07.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:26:07.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:07.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_47 128M 2026-02-20T21:26:07.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:07.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:07.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:07.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:07.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:26:07.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:07.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:26:07.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:26:07.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:26:07.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:07.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_47 2026-02-20T21:26:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:26:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:26:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:26:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:26:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:26:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_47 2026-02-20T21:26:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:26:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:26:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=9 2026-02-20T21:26:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:26:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_47 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:26:07.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 23076864 bytes 2026-02-20T21:26:08.461 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:08.457+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:26:08.461 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:08.457+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:26:08.461 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:08.457+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:26:08.462 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:08.458+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:08.462 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:08.458+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:08.462 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:08.458+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:08.462 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:08.458+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:08.496 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:08.492+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:26:08.496 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:08.492+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:26:08.497 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:08.493+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:08.497 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:08.493+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:08.498 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:08.494+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:26:08.498 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:08.494+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:08.498 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:08.494+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:08.498 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:08.494+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:08.498 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:08.494+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:08.498 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:08.494+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:08.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 77586432 bytes 2026-02-20T21:26:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 84364288 bytes 2026-02-20T21:26:10.030 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.026+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:10.031 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.027+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:10.033 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.030+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:10.033 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.030+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:10.037 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.033+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:26:10.039 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.036+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:26:10.040 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.036+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:26:10.041 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.037+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:10.041 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.037+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:26:10.041 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.037+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:10.043 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.040+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:10.043 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.040+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:10.219 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.215+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:26:10.219 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.215+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:10.219 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.215+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:10.219 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.215+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:10.665 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.662+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:26:10.666 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.663+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:26:10.667 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.663+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:10.667 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.663+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:26:10.668 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.664+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:10.671 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.667+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:10.671 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.667+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:10.671 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.667+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:10.671 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:10.667+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:10.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 168803328 bytes 2026-02-20T21:26:11.542 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:11.538+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:26:11.542 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:11.538+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:26:11.542 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:11.538+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:11.543 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:11.538+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:26:11.543 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:11.538+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:11.543 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:11.538+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:11.543 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:11.538+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:12.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 207934464 bytes 2026-02-20T21:26:12.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 241289216 bytes 2026-02-20T21:26:13.023 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:13.019+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:26:13.024 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:13.021+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:26:13.025 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:13.021+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:13.025 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:13.021+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:13.025 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:13.021+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:26:13.026 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:13.022+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:13.026 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:13.022+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:13.027 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:13.022+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:13.027 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:13.022+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:13.027 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:13.022+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:13.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 266908672 bytes 2026-02-20T21:26:13.360 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:13.356+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:13.360 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:13.356+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:13.360 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:13.356+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:13.361 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:13.357+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:13.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 275545088 bytes 2026-02-20T21:26:13.440 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:13.436+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:13.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 303716352 bytes 2026-02-20T21:26:13.861 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:13.857+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9110 2026-02-20T21:26:13.861 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:13.857+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:26:13.862 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:13.857+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:26:14.126 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:14.122+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:14.337 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:14.333+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 355386368 bytes 2026-02-20T21:26:14.949 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:14.945+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:14.951 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:14.947+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:14.951 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:14.947+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:14.952 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:14.948+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:14.955 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:14.951+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:14.955 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:14.951+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:15.116 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:15.112+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:15.116 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:15.112+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 383731712 bytes 2026-02-20T21:26:15.475 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:15.471+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:26:15.476 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:15.471+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:26:15.476 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:15.471+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:26:15.476 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:15.471+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:15.476 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:15.471+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:26:15.476 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:15.471+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:15.476 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:15.471+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:15.476 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:15.471+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:16.076 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.072+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e21464e0 2026-02-20T21:26:16.077 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.073+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:26:16.077 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.073+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:26:16.077 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.073+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:26:16.077 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.073+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:26:16.077 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.073+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:26:16.156 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.152+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:26:16.158 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.154+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f908f0 2026-02-20T21:26:16.158 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.154+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:26:16.158 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.154+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:26:16.158 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.154+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:26:16.159 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.154+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:26:16.159 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.154+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:16.159 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.154+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:26:16.159 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.154+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:16.159 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.154+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:16.159 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.154+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 438475776 bytes 2026-02-20T21:26:16.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:26:16.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:26:16.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:26:16.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:26:16.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:26:16.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_48 2026-02-20T21:26:16.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_48 journal 128M 2026-02-20T21:26:16.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:16.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:16.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:16.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:16.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:26:16.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:16.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:26:16.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:26:16.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_48 128M 2026-02-20T21:26:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:16.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:26:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:26:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:26:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:26:16.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:16.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_48 2026-02-20T21:26:16.516 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.512+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:26:16.518 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.514+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:26:16.518 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.514+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:26:16.518 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.514+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:26:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:26:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:26:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:26:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:26:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:26:16.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_48 2026-02-20T21:26:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:26:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:26:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-20T21:26:16.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:26:16.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_48 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:26:16.736 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.731+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:26:16.736 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.731+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:26:16.736 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.731+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:26:16.736 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:16.731+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:26:16.804 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:16.800+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:26:16.804 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:16.800+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:17.241 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:17.237+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:26:17.243 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:17.239+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f908f0 2026-02-20T21:26:17.244 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:17.239+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:26:17.244 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:17.239+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:26:17.244 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:17.240+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:26:17.245 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:17.241+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:26:17.245 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:17.241+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:17.245 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:17.242+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:26:17.246 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:17.242+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:17.246 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:17.242+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:17.246 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:17.242+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:17.420 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.416+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:17.422 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.418+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:17.425 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.421+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:17.425 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.421+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:17.425 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.421+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:17.631 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.628+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:17.631 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.628+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:17.632 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.629+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:26:17.632 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.629+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:17.632 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.629+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:17.633 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.629+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:17.633 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.629+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:17.633 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:17.629+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:17.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 57538560 bytes 2026-02-20T21:26:18.162 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.158+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:26:18.163 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.160+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:26:18.164 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.160+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:26:18.164 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.160+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:26:18.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 119589888 bytes 2026-02-20T21:26:18.527 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.523+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:18.529 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.525+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:18.529 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.525+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:18.551 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.547+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:18.551 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.548+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:26:18.553 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.549+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:18.555 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.551+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:18.555 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.551+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:18.623 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.618+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:26:18.623 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.618+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:26:18.624 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.619+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:26:18.649 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:18.645+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:26:18.649 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:18.645+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:18.649 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:18.645+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:18.649 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:18.645+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:18.664 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.660+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:18.666 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:18.662+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:19.380 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:19.376+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:26:19.380 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:19.376+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:26:19.380 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:19.376+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:26:19.666 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:19.662+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:26:19.668 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:19.664+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:26:19.668 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:19.664+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:20.126 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.122+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:26:20.126 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.122+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:26:20.127 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.122+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:20.127 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.122+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:20.127 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.122+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:20.127 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.122+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:20.360 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.356+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:26:20.363 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.359+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:20.363 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.359+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:20.363 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.359+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:20.364 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.361+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 198177792 bytes 2026-02-20T21:26:20.586 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:20.678 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:20.674+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fde4e0 2026-02-20T21:26:20.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:20.676+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441450 2026-02-20T21:26:20.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:20.676+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:26:20.681 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:20.677+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d790 2026-02-20T21:26:20.681 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:20.677+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:26:20.681 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:20.677+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:26:20.694 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.690+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:26:20.694 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.690+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:26:20.694 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.690+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:20.695 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.691+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:20.695 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.691+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:20.695 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:20.691+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:21.058 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.052+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:26:21.058 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.052+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:21.058 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.053+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:21.058 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.053+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:21.058 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.053+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:21.144 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.140+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1724e0 2026-02-20T21:26:21.147 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.143+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:26:21.147 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.143+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:26:21.147 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.143+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:21.150 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.146+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:26:21.150 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.146+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:21.150 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.146+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:21.150 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.146+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:26:21.150 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.146+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:21.152 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.148+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:26:21.202 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.198+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:21.202 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.198+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:21.203 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.198+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:21.203 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.198+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:21.258 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.254+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:21.259 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.255+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:21.262 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.258+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:26:21.262 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.258+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:21.262 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.258+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:21.262 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.258+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:21.262 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.258+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:21.262 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.258+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:21.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 245695488 bytes 2026-02-20T21:26:21.454 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.450+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:21.454 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.450+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:21.454 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.450+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:21.454 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.450+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:21.454 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.450+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:21.555 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:21.551+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:21.789 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.785+0000 7fd67aee6640 -1 reset not still connected to 0x55daae439ad0 2026-02-20T21:26:21.789 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.785+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:26:21.790 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.786+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:26:21.790 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:21.786+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:22.396 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.392+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:22.396 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.392+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:22.396 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.392+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:22.397 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.392+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:22.397 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.392+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:22.397 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.392+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:22.408 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.404+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:22.411 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.407+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:26:22.411 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.407+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:22.411 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.407+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:22.411 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.407+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:22.497 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.493+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:22.497 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.493+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:22.502 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.498+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:22.502 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.498+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:22.502 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.498+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:22.502 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.498+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:22.502 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:22.498+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:22.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 315588608 bytes 2026-02-20T21:26:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 373474304 bytes 2026-02-20T21:26:23.304 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:23.300+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:26:23.306 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:23.303+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:26:23.732 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:23.728+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:26:23.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 427857920 bytes 2026-02-20T21:26:23.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 428461056 bytes 2026-02-20T21:26:24.400 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:24.397+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:26:24.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 468564992 bytes 2026-02-20T21:26:24.529 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:24.525+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:26:24.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:26:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:26:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:26:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:26:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:26:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_49 2026-02-20T21:26:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_49 journal 128M 2026-02-20T21:26:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:24.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:26:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:26:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:26:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_49 128M 2026-02-20T21:26:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:24.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:26:24.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:24.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:26:24.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:26:24.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:26:24.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:24.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_49 2026-02-20T21:26:24.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:26:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:26:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:26:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:26:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:26:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_49 2026-02-20T21:26:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:24.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:26:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:26:24.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=9 2026-02-20T21:26:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:26:24.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_49 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:26:24.841 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:24.837+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:26:24.842 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:24.837+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e21464e0 2026-02-20T21:26:24.842 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:24.837+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:26:24.842 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:24.837+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:26:24.844 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:24.840+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:26:24.844 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:24.840+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:26:25.331 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:25.327+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:26:25.331 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:25.327+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:25.333 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:25.329+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:25.333 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:25.329+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:25.333 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:25.329+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:25.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 46603264 bytes 2026-02-20T21:26:26.254 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:26.250+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:26.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 91245568 bytes 2026-02-20T21:26:26.565 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:26.561+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:26:26.565 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:26.561+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:26:26.566 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:26.561+0000 7f413d8ae640 -1 reset not still connected to 0x55d270ea0f70 2026-02-20T21:26:26.566 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:26.561+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:26:27.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 152437760 bytes 2026-02-20T21:26:27.524 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.520+0000 7fc9d03ed640 -1 reset not still connected to 0x5576df3755f0 2026-02-20T21:26:27.526 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.522+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:26:27.529 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.525+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f908f0 2026-02-20T21:26:27.529 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.525+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:26:27.531 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.528+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:26:27.531 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.528+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:26:27.532 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.528+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:26:27.534 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.530+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:27.534 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.530+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:26:27.534 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.530+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:27.535 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.530+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:27.535 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.530+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e16c0 2026-02-20T21:26:27.535 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:27.530+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:27.810 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.806+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:27.810 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.807+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:27.810 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.807+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:27.930 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.925+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:26:27.930 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.925+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:26:27.930 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.925+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:27.930 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.925+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:26:27.932 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.929+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:27.933 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.929+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:27.933 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.929+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:27.936 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.932+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:27.936 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.932+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:26:27.936 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.932+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:27.936 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:27.932+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:26:28.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 166930432 bytes 2026-02-20T21:26:28.532 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:28.528+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3555d40 2026-02-20T21:26:28.532 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:28.528+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:26:28.533 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:28.528+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:26:28.533 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:28.528+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:26:29.222 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.218+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:26:29.223 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.218+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:26:29.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:26:29.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.220+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:26:29.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:29.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:26:29.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:29.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:29.225 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:29.225 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:29.283 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.279+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:29.283 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.279+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:29.286 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.281+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:29.286 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.281+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:29.286 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.281+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:29.286 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.281+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:29.288 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.284+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:29.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 248310784 bytes 2026-02-20T21:26:29.478 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.473+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:26:29.478 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.473+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef380 2026-02-20T21:26:29.478 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.473+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:26:29.479 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.474+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:26:29.480 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.475+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:26:29.480 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.476+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:29.480 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.476+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:26:29.481 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.476+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:29.481 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.476+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:29.481 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.476+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:29.481 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.476+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 268369920 bytes 2026-02-20T21:26:29.793 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.789+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:26:29.794 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.790+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef380 2026-02-20T21:26:29.794 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.790+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:26:29.796 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.792+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:26:29.797 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.793+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:26:29.797 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.793+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:29.797 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.793+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:26:29.797 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.793+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:29.798 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.794+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:29.798 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.794+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:29.798 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.794+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:29.916 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.912+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423ad0 2026-02-20T21:26:29.916 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.912+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef2b0 2026-02-20T21:26:29.919 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.916+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:29.919 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.916+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:29.922 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.918+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:26:29.925 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.921+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:29.925 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.921+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:29.925 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.921+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:29.926 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.921+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:29.926 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:29.921+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:30.613 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:30.609+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e21464e0 2026-02-20T21:26:30.615 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:30.611+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:26:30.615 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:30.611+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:26:30.616 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:30.612+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:26:30.616 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:30.612+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:26:30.616 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:30.612+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:26:30.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 346720256 bytes 2026-02-20T21:26:31.063 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:31.059+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:31.251 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:31.247+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:26:31.252 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:31.248+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:26:31.252 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:31.248+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:26:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 378115072 bytes 2026-02-20T21:26:31.578 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:31.575+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1c06d00 2026-02-20T21:26:31.580 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:31.576+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2147380 2026-02-20T21:26:31.580 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:31.576+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2588c30 2026-02-20T21:26:31.580 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:31.576+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351e8f0 2026-02-20T21:26:31.580 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:31.576+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3555d40 2026-02-20T21:26:31.580 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:31.576+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:26:31.581 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:31.577+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0410 2026-02-20T21:26:31.582 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:31.578+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:26:31.583 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:31.579+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:26:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 406447104 bytes 2026-02-20T21:26:32.174 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:32.170+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:26:32.176 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:32.172+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:32.178 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:32.174+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:26:32.178 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:32.174+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:32.178 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:32.174+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:32.178 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:32.174+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:32.454 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:32.450+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:32.454 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:32.450+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:32.455 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:32.451+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:32.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 436481024 bytes 2026-02-20T21:26:33.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 473335808 bytes 2026-02-20T21:26:33.464 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:33.460+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:26:33.464 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:33.460+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:33.464 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:33.460+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:26:33.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:26:33.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:26:33.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:26:33.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:26:33.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:26:33.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_50 2026-02-20T21:26:33.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image_and_enable_mirror cluster2 mirror image_50 journal 128M 2026-02-20T21:26:33.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:33.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:33.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:33.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:33.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:26:33.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:26:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n journal ']' 2026-02-20T21:26:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_50 128M 2026-02-20T21:26:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:26:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:26:33.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 128M ']' 2026-02-20T21:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=128M 2026-02-20T21:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:26:33.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 128M --image-feature layering,exclusive-lock,journaling mirror/image_50 2026-02-20T21:26:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ is_pool_mirror_mode_image mirror 2026-02-20T21:26:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' pool = image ']' 2026-02-20T21:26:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ case "${pool}" in 2026-02-20T21:26:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 1 2026-02-20T21:26:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n '' ']' 2026-02-20T21:26:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_50 2026-02-20T21:26:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:26:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:26:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-20T21:26:33.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:26:33.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_50 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:26:33.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 11840512 bytes 2026-02-20T21:26:34.062 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:34.058+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2588c30 2026-02-20T21:26:34.062 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:34.058+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351e8f0 2026-02-20T21:26:34.062 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:34.058+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3555d40 2026-02-20T21:26:34.062 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:34.058+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:26:34.065 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:34.061+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0410 2026-02-20T21:26:34.066 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:34.061+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:26:34.066 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:34.061+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:26:34.097 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:34.093+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:34.097 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:34.093+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:34.097 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:34.093+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:34.098 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:34.093+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:34.405 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:34.401+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:26:34.405 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:34.401+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:34.406 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:34.401+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:34.406 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:34.401+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:34.406 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:34.401+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:34.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 39769088 bytes 2026-02-20T21:26:34.721 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:34.717+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351e8f0 2026-02-20T21:26:34.722 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:34.718+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:26:34.722 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:34.718+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:26:34.722 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:34.718+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:26:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 81238016 bytes 2026-02-20T21:26:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 88778752 bytes 2026-02-20T21:26:35.263 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.260+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:35.351 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.347+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:26:35.351 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.347+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:26:35.352 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.347+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:26:35.352 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.347+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:35.353 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.349+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:35.354 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.350+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:35.354 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.350+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:35.354 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.350+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:35.355 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.350+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:35.355 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.351+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:26:35.355 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.351+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:35.355 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.351+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:26:35.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 124679168 bytes 2026-02-20T21:26:35.906 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.902+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:26:35.906 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.902+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:26:35.906 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.902+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:35.909 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.905+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:35.910 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.906+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:35.910 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.906+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:35.910 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.906+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:35.912 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.908+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:35.912 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:35.908+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:35.948 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:35.944+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:26:35.948 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:35.944+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:26:35.948 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:35.944+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:26:35.949 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:35.944+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:26:35.950 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:35.947+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:26:35.950 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:35.947+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:35.951 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:35.947+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:26:35.951 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:35.947+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:35.951 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:35.947+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:35.951 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:35.947+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:36.088 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:36.084+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:26:36.090 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:36.085+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:26:36.090 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:36.085+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:26:36.090 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:36.086+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:26:36.090 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:36.086+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:36.090 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:36.086+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:26:36.280 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:36.275+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:36.280 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:36.275+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:36.558 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:36.554+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f908f0 2026-02-20T21:26:36.558 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:36.554+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:26:36.560 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:36.556+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:26:36.561 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:36.556+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:26:36.561 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:36.556+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:26:36.561 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:36.556+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:36.562 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:36.557+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:26:36.562 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:36.557+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:36.562 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:36.557+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2ba1380 2026-02-20T21:26:36.562 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:36.557+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:36.562 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:36.557+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e16c0 2026-02-20T21:26:36.562 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:36.557+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:36.768 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:36.764+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:36.769 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:36.764+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:36.769 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:36.765+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:36.886 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:36.882+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:36.887 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:36.883+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:26:36.925 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:36.921+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:36.925 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:36.921+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:36.925 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:36.921+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:36.925 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:36.921+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:36.925 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:36.921+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:37.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 178647040 bytes 2026-02-20T21:26:37.232 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:37.228+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:37.314 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:37.311+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:26:37.319 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:37.315+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:37.320 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:37.316+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:26:37.320 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:37.316+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:37.320 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:37.316+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:37.320 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:37.316+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:37.320 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:37.316+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:37.567 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:37.563+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:26:37.569 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:37.565+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:26:37.570 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:37.566+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:26:37.571 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:37.567+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:26:38.131 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:38.127+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:26:38.132 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:38.127+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:26:38.132 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:38.127+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:26:38.132 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:38.127+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:26:38.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 232433664 bytes 2026-02-20T21:26:38.373 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.369+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:38.375 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.371+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:26:38.375 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.371+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:38.375 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.371+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:38.376 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.373+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:38.377 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.373+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:38.378 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.374+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:26:38.378 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.374+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:38.378 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.374+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:26:38.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 270319616 bytes 2026-02-20T21:26:38.754 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.750+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:38.755 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.752+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:26:38.756 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.752+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:38.756 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.752+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:38.756 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.752+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:38.873 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.869+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:26:38.873 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.869+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:38.874 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.869+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:26:38.874 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.869+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:38.874 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.869+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:38.874 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:38.869+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:39.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 277590016 bytes 2026-02-20T21:26:39.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 290822144 bytes 2026-02-20T21:26:39.639 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:39.635+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:26:39.639 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:39.635+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:26:39.639 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:39.635+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:26:39.639 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:39.635+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:26:39.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 317981696 bytes 2026-02-20T21:26:40.228 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:40.224+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:26:40.229 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:40.225+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:26:40.230 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:40.225+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:40.230 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:40.225+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:26:40.230 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:40.226+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:40.231 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:40.227+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:40.232 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:40.228+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e16c0 2026-02-20T21:26:40.232 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:40.228+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:40.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 351835136 bytes 2026-02-20T21:26:40.472 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:40.468+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:26:40.472 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:40.468+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:40.472 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:40.468+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:26:40.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 371005440 bytes 2026-02-20T21:26:40.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:26:40.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:26:40.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:26:40.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:26:40.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_pool_images cluster2 mirror 50 2026-02-20T21:26:40.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:40.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:40.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_count=50 2026-02-20T21:26:40.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:26:40.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local count 2026-02-20T21:26:40.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_count=0 2026-02-20T21:26:40.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:26:40.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 40 2026-02-20T21:26:40.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-20T21:26:40.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 1 -ne 1 2026-02-20T21:26:40.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-20T21:26:40.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'images: ' 2026-02-20T21:26:40.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 2 2026-02-20T21:26:40.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T21:26:40.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ count=50 2026-02-20T21:26:40.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 50 = 50 2026-02-20T21:26:40.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:26:40.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_pool_healthy cluster2 mirror 2026-02-20T21:26:40.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:26:40.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:40.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:26:40.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state 2026-02-20T21:26:40.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 40 2026-02-20T21:26:40.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-20T21:26:40.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 1 -ne 1 2026-02-20T21:26:40.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 -p mirror mirror pool status 2026-02-20T21:26:40.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'image health:' 2026-02-20T21:26:40.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 3 2026-02-20T21:26:40.735 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:40.731+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef380 2026-02-20T21:26:40.735 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:40.731+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:26:40.738 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:40.733+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:26:40.738 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:40.733+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:40.739 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:40.735+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:40.739 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:40.735+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:40.739 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:40.735+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:40.740 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:40.735+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:40.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T21:26:40.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=OK 2026-02-20T21:26:40.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test OK = ERROR 2026-02-20T21:26:40.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test OK = OK 2026-02-20T21:26:40.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:26:40.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_pool_images cluster1 mirror 50 2026-02-20T21:26:40.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:26:40.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:40.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_count=50 2026-02-20T21:26:40.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:26:40.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local count 2026-02-20T21:26:40.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_count=0 2026-02-20T21:26:40.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:26:40.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 40 2026-02-20T21:26:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-20T21:26:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 1 -ne 1 2026-02-20T21:26:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-20T21:26:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'images: ' 2026-02-20T21:26:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 2 2026-02-20T21:26:40.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T21:26:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ count=50 2026-02-20T21:26:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 50 = 50 2026-02-20T21:26:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:26:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_pool_healthy cluster1 mirror 2026-02-20T21:26:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:26:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:26:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:26:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state 2026-02-20T21:26:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 40 2026-02-20T21:26:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-20T21:26:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 1 -ne 1 2026-02-20T21:26:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-20T21:26:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'image health:' 2026-02-20T21:26:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 3 2026-02-20T21:26:40.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T21:26:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=WARNING 2026-02-20T21:26:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test WARNING = ERROR 2026-02-20T21:26:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test WARNING = OK 2026-02-20T21:26:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-20T21:26:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 -ne 1 2026-02-20T21:26:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 30 2026-02-20T21:26:41.046 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.042+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:26:41.049 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.045+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f908f0 2026-02-20T21:26:41.050 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.045+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:26:41.050 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.045+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:26:41.050 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.045+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:26:41.051 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.047+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:26:41.051 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.047+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:41.051 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.047+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:26:41.051 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.047+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2ba1380 2026-02-20T21:26:41.053 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.049+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:41.053 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.049+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e16c0 2026-02-20T21:26:41.053 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.049+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:41.072 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.067+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:26:41.072 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.068+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:26:41.072 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.068+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:26:41.072 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.068+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:26:41.072 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.068+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:41.073 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.069+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:26:41.077 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.073+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2ba1380 2026-02-20T21:26:41.078 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.074+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e16c0 2026-02-20T21:26:41.078 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:41.074+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:41.589 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:41.585+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:26:41.590 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:41.586+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:26:41.592 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:41.588+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:26:41.592 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:41.588+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:41.593 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:41.588+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:26:42.040 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:42.035+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:26:42.040 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:42.036+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:26:42.041 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:42.038+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:26:42.043 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:42.039+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:26:42.197 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:42.193+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:26:42.197 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:42.193+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:26:42.202 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:42.198+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:42.202 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:42.198+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:26:42.722 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.718+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:26:42.722 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.718+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:26:42.723 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.719+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:26:42.724 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.720+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:42.724 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.720+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:26:42.724 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.720+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:42.724 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.720+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:42.724 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.720+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:42.725 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.720+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:42.725 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.720+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:42.726 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.722+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:26:42.726 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.722+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:42.726 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:42.722+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:26:43.758 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:43.754+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3143450 2026-02-20T21:26:43.759 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:43.755+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:26:43.760 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:43.756+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb3860 2026-02-20T21:26:43.760 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:43.756+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:26:43.760 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:43.756+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:26:43.760 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:43.756+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:26:43.760 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:43.756+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:26:43.940 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:43.936+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:26:43.941 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:43.937+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:26:44.110 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.106+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1724e0 2026-02-20T21:26:44.112 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.108+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:26:44.112 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.108+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:26:44.112 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.108+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:26:44.112 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.108+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:44.113 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.109+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:26:44.113 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.109+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:44.113 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.109+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:44.113 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.109+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:44.114 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.109+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:44.114 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.109+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:44.114 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.109+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:44.114 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.110+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:26:44.150 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.146+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:44.150 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.146+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:44.150 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.146+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:44.150 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.146+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:44.151 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.146+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:44.151 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.146+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:44.185 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.181+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92bc70 2026-02-20T21:26:44.415 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.411+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:26:44.415 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.411+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:26:44.416 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.412+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:26:44.417 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.413+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:44.417 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.413+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:44.417 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.413+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:44.417 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.413+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:44.417 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.413+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:44.419 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:26:44.419 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:44.424 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.420+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:26:44.452 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.448+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:26:44.453 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.448+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:26:44.453 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.448+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:26:44.453 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.448+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:44.453 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.448+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:44.453 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.448+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:44.453 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.448+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:44.453 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.448+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:44.453 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:44.448+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:45.004 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:45.000+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:26:45.004 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:45.000+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:45.004 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:45.000+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:26:45.095 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:45.091+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:45.096 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:45.093+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:45.097 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:45.094+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:45.266 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:45.262+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:45.267 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:45.263+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:45.267 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:45.263+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:45.268 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:45.264+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:45.268 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:45.264+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:45.268 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:45.264+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:45.329 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:45.325+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:45.330 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:45.326+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e16c0 2026-02-20T21:26:45.330 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:45.326+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:45.384 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:45.380+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:45.613 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:45.609+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:26:45.613 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:45.609+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:26:45.617 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:45.613+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:26:45.617 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:45.613+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:26:45.618 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:45.615+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:26:45.746 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:45.740+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:26:45.746 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:45.740+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:26:45.746 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:45.740+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:26:45.747 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:45.740+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:45.747 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:45.741+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:45.747 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:45.742+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:46.076 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:46.071+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:26:46.076 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:46.071+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:26:46.331 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:46.327+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:46.331 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:46.327+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:46.427 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.421+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:46.427 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.422+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:46.427 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.422+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:46.427 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.423+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:26:46.449 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.444+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b8dd0 2026-02-20T21:26:46.449 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.444+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9110 2026-02-20T21:26:46.449 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.444+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:26:46.449 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.444+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:26:46.577 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.573+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:46.578 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.574+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:46.578 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.574+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:46.581 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.577+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1724e0 2026-02-20T21:26:46.582 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.578+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:26:46.582 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.578+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:46.586 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:46.586 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:46.586 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:46.586 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:46.586 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:46.586 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:46.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:47.165 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:47.160+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:26:47.166 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:47.161+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:26:47.166 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:47.161+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:26:47.166 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:47.161+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:26:47.167 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:47.162+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:47.167 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:47.162+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:47.168 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:47.163+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:47.168 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:47.164+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:47.422 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:47.417+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:47.422 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:47.417+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:26:47.422 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:47.418+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:47.422 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:47.418+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:47.422 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:47.418+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:47.422 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:47.418+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:47.425 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:47.421+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:47.425 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:47.421+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:26:47.425 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:47.421+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:47.426 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:47.422+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:26:48.169 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:48.165+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:26:48.169 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:48.165+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:26:48.171 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:48.168+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:26:48.172 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:48.169+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:26:48.172 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:48.169+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:26:48.322 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:48.318+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:48.322 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:48.318+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:48.322 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:48.318+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:48.322 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:48.318+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:48.322 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:48.318+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:48.323 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:48.318+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:48.740 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:48.736+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:48.740 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:48.737+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:48.740 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:48.737+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:48.740 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:48.737+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:48.831 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:48.827+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:26:49.238 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.234+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:49.238 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.234+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:49.238 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.234+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:49.259 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.4705721769044482) > weight (0.25) 2026-02-20T21:26:49.259 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.33582429678323944) > weight (0.25) 2026-02-20T21:26:49.259 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.8038578633833239) > weight (0.25) 2026-02-20T21:26:49.259 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.7299173410068359) > weight (0.25) 2026-02-20T21:26:49.259 INFO:tasks.rbd_mirror_thrash:waiting for 88.56258074058844 secs before thrashing 2026-02-20T21:26:49.499 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.495+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:26:49.501 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.497+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:49.501 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.497+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:49.502 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.498+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:26:49.502 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.498+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:49.502 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.498+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:49.502 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.498+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:49.502 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.498+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:49.502 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.498+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:49.621 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.617+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:49.622 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:49.617+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:51.053 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.049+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:26:51.053 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.049+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:51.054 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.049+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:51.054 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.049+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:51.054 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.049+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:51.054 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.049+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d040 2026-02-20T21:26:51.054 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.049+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:26:51.054 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.049+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:26:51.054 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.049+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:26:51.055 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.050+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:26:51.055 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.050+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:51.055 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.050+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:51.055 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.050+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:51.073 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.070+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:26:51.079 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.075+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:51.079 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.075+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:51.386 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55daad85d1e0 2026-02-20T21:26:51.386 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55daae4380d0 2026-02-20T21:26:51.386 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9badd0 2026-02-20T21:26:51.386 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf1f8340 2026-02-20T21:26:51.386 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423ad0 2026-02-20T21:26:51.386 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:26:51.386 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef2b0 2026-02-20T21:26:51.386 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:26:51.386 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:51.387 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:51.387 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:26:51.387 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:51.387 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:51.387 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.381+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:51.387 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.383+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:51.387 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.383+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:51.650 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.646+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:51.650 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.646+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:51.652 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.648+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:51.654 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.650+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:51.734 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.730+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:51.735 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.731+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:51.737 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:51.734+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:51.959 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:51.955+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:26:52.193 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:52.189+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a60340 2026-02-20T21:26:52.194 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:52.190+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:52.194 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:52.190+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:52.194 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:52.190+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:26:52.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:52.190+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:52.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:52.190+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:52.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:52.190+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:52.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:52.190+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:52.195 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:52.190+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:52.271 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.267+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1724e0 2026-02-20T21:26:52.273 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.269+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:26:52.273 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.269+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:26:52.273 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.269+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:26:52.273 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.269+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:52.273 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.269+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:52.273 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.269+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:52.273 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.269+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:52.273 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.269+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:52.273 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.269+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:52.273 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.269+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:52.407 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.403+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1724e0 2026-02-20T21:26:52.409 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.405+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:26:52.409 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.405+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:26:52.409 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.405+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:26:52.409 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.405+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:52.409 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.406+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:52.410 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.407+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:52.410 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.407+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:52.410 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.407+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:52.410 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.407+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:52.411 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.407+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:52.420 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:52.416+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:26:52.420 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:52.416+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:52.420 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:52.416+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:26:52.493 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.489+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:52.495 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.491+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:26:52.495 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.491+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:52.495 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.491+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:52.495 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.491+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:52.601 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.597+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1724e0 2026-02-20T21:26:52.603 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.599+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:26:52.603 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.599+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:26:52.603 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.599+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:26:52.604 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.599+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:26:52.604 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.599+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:52.604 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.599+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:52.604 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.599+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:52.604 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.599+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:52.604 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.599+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:52.605 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.601+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:52.764 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.760+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:26:52.766 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.762+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:26:52.766 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.762+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:26:52.766 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:52.762+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:26:53.383 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:53.379+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:26:53.386 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:53.382+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:26:53.387 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:53.383+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:26:53.387 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:53.383+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:26:53.388 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:53.384+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:26:53.388 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:53.384+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:53.388 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:53.384+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:53.388 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:53.384+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e16c0 2026-02-20T21:26:53.388 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:53.384+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:53.430 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:53.426+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:53.430 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:53.426+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:53.430 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:53.426+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:53.432 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:53.428+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:53.432 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:53.428+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:53.967 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:53.964+0000 7f413d8ae640 -1 reset not still connected to 0x55d270ea0f70 2026-02-20T21:26:53.969 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:53.965+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b8dd0 2026-02-20T21:26:53.969 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:53.965+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9110 2026-02-20T21:26:53.969 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:53.965+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:26:53.969 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:53.965+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:26:54.128 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:54.124+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:26:54.128 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:54.124+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:54.128 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:54.124+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:26:54.128 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:54.124+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:26:54.128 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:54.124+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:54.129 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:54.126+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:54.129 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:54.126+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:54.255 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:54.251+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:54.255 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:54.251+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:54.256 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:54.252+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:54.257 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:54.253+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:54.257 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:54.253+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:54.262 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:54.258+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:26:55.108 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.105+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:26:55.109 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.105+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:26:55.223 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.219+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:26:55.223 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.219+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:26:55.223 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.219+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:26:55.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.219+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:26:55.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.219+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:26:55.247 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.243+0000 7fd67aee6640 -1 reset not still connected to 0x55daafea6000 2026-02-20T21:26:55.248 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.244+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423040 2026-02-20T21:26:55.249 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.245+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2d2e8f0 2026-02-20T21:26:55.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.246+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fde4e0 2026-02-20T21:26:55.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.246+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441450 2026-02-20T21:26:55.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.246+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:26:55.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.246+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d790 2026-02-20T21:26:55.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.246+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:26:55.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.246+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:26:55.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.246+0000 7fd67aee6640 -1 reset not still connected to 0x55dae421fd40 2026-02-20T21:26:55.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.246+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:26:55.587 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:55.583+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:26:55.957 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:55.953+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:26:55.957 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:55.953+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:26:55.958 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:55.954+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:26:55.959 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:55.956+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:26:55.960 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:55.956+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:26:55.960 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:55.956+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:26:56.053 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:56.049+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:26:56.053 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:56.049+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e21464e0 2026-02-20T21:26:56.053 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:56.049+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:26:56.053 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:56.049+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:26:56.054 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:56.050+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:26:56.054 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:56.050+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:26:56.915 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:56.911+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:26:56.915 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:56.911+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:56.918 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:56.914+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:56.918 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:56.914+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:56.918 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:56.915+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:26:56.919 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:56.915+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:26:56.919 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:56.915+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:56.919 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:56.915+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:26:57.311 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:57.307+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:26:57.311 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:57.307+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:26:57.311 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:57.307+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:26:57.314 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:57.310+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:26:57.314 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:57.310+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:26:57.314 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:26:57.310+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:26:57.916 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:57.912+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:26:57.918 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:57.914+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:26:57.918 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:57.914+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:26:57.919 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:57.915+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:26:58.158 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:58.154+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:26:58.159 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:58.154+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:26:58.159 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:58.155+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef380 2026-02-20T21:26:58.159 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:58.155+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:26:58.160 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:58.156+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:26:58.160 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:58.156+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:26:58.160 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:58.156+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:26:58.161 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:58.156+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:26:58.161 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:58.156+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:26:58.161 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:58.158+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:26:58.162 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:58.158+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:26:58.162 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:26:58.158+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:26:58.480 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:58.476+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:26:58.481 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:58.477+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:26:58.482 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:58.478+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e16c0 2026-02-20T21:26:58.482 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:58.478+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:26:59.009 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.004+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e14abc70 2026-02-20T21:26:59.009 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.004+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:26:59.010 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.006+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e21464e0 2026-02-20T21:26:59.010 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.006+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:26:59.010 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.006+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:26:59.010 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.006+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:26:59.010 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.006+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:26:59.010 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.006+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:26:59.458 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.455+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:26:59.460 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.456+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:26:59.460 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.457+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:26:59.461 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.457+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:26:59.462 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:26:59.458+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:27:00.157 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.153+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:00.158 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.154+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:27:00.158 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.154+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:00.159 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.155+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:27:00.159 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.155+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:00.159 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.155+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:00.160 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.156+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:00.162 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.159+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:00.162 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.159+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:00.163 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.159+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:27:00.163 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.159+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:00.163 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.159+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:00.163 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.159+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:00.163 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.159+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:00.163 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.159+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:00.163 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.160+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:00.164 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.160+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:00.164 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.160+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:00.194 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.190+0000 7fd67aee6640 -1 reset not still connected to 0x55daafea6000 2026-02-20T21:27:00.197 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.193+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423040 2026-02-20T21:27:00.197 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.193+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2d2e8f0 2026-02-20T21:27:00.197 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.193+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fde4e0 2026-02-20T21:27:00.197 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.193+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441450 2026-02-20T21:27:00.199 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.195+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:27:00.200 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.195+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d790 2026-02-20T21:27:00.200 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.195+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:27:00.200 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.195+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:27:00.200 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.195+0000 7fd67aee6640 -1 reset not still connected to 0x55dae421fd40 2026-02-20T21:27:00.200 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.195+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:27:00.760 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.756+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:00.760 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.756+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:00.761 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.758+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:00.762 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.759+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:00.763 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:00.759+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:00.780 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.777+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:27:00.781 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.777+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:27:00.781 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.777+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:27:00.781 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.777+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:27:00.781 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:00.777+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:01.721 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:01.717+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:27:01.721 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:01.717+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:27:01.721 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:01.717+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:27:02.840 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:02.836+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:27:02.840 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:02.836+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:27:02.840 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:02.836+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:03.009 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.005+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:27:03.010 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.005+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:27:03.010 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.007+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:03.011 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.007+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:03.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.441+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:27:03.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.441+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:27:03.446 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.442+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:27:03.446 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.442+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:27:03.446 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.442+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:27:03.447 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.442+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:27:03.447 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.443+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:27:03.447 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.443+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:27:03.480 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.476+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:03.481 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.478+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:27:03.482 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.478+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:03.483 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.479+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:27:03.484 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.480+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:03.484 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.480+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:03.484 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.480+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:03.484 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.480+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:03.690 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.685+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:27:03.690 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.685+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:27:03.692 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.688+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:27:03.693 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.688+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:27:03.693 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.688+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:27:03.693 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.688+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:27:03.693 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:03.688+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:27:04.237 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.233+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:04.238 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.233+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:04.238 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.233+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:04.238 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.233+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:04.238 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.233+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:04.238 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.233+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:04.555 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.549+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:27:04.556 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.552+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:27:04.556 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.552+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:27:04.556 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.552+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:27:04.557 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.552+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107a1a0 2026-02-20T21:27:04.557 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.552+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:27:04.557 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.552+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:27:04.557 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.552+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:27:04.557 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.552+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:04.803 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.799+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:04.803 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.799+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:04.803 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.799+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:04.803 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.799+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:04.804 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.799+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:04.804 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.800+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:04.804 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.800+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:04.805 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:04.800+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:05.416 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:05.412+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:05.420 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:05.416+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:05.420 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:05.416+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:05.421 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:05.416+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:05.421 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:05.416+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:05.421 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:05.418+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:05.421 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:05.418+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:05.422 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:05.418+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:05.443 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:05.439+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:27:05.443 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:05.439+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:27:05.444 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:05.441+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:27:05.445 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:05.441+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:27:05.445 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:05.441+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:27:05.919 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:05.915+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:27:05.919 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:05.915+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:06.044 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.040+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:27:06.044 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.040+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:27:06.045 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.040+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:06.045 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.040+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:27:06.045 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.040+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:06.045 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.040+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:06.045 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.040+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:06.045 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.040+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:06.046 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.042+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:06.046 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.042+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:06.046 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.042+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:06.046 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.042+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:06.511 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:06.507+0000 7fd67aee6640 -1 reset not still connected to 0x55daafea6000 2026-02-20T21:27:06.512 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:06.508+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fde4e0 2026-02-20T21:27:06.512 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:06.508+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441450 2026-02-20T21:27:06.513 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:06.509+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:27:06.514 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:06.510+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d790 2026-02-20T21:27:06.514 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:06.510+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:27:06.514 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:06.510+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:27:06.514 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:06.510+0000 7fd67aee6640 -1 reset not still connected to 0x55dae421fd40 2026-02-20T21:27:06.514 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:06.510+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:27:06.529 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.525+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:06.530 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.525+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:06.530 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.527+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:06.532 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:06.528+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:06.615 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:06.611+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:07.156 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:07.151+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef380 2026-02-20T21:27:07.156 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:07.151+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:07.156 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:07.151+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:07.156 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:07.151+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:07.156 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:07.151+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:07.158 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:07.154+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:07.377 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:07.373+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:27:07.377 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:07.373+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:27:07.377 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:07.373+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:27:07.379 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:07.374+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:27:07.379 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:07.374+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:27:07.379 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:07.374+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:27:07.379 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:07.374+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:07.650 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:07.646+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:27:08.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.419+0000 7fd67aee6640 -1 reset not still connected to 0x55daafea6000 2026-02-20T21:27:08.425 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.421+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2d2e8f0 2026-02-20T21:27:08.425 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.421+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441450 2026-02-20T21:27:08.425 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.421+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:27:08.427 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.423+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d790 2026-02-20T21:27:08.427 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.423+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:27:08.427 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.423+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:27:08.427 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.423+0000 7fd67aee6640 -1 reset not still connected to 0x55dae421fd40 2026-02-20T21:27:08.428 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.423+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:27:08.607 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.603+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:08.609 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.605+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:27:08.610 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.606+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:27:08.611 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.606+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:08.611 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.606+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:27:08.611 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.606+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:08.612 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.608+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:08.612 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.608+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:08.612 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.608+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:08.731 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.727+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:08.733 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.729+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:27:08.733 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.729+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:08.733 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.729+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:08.733 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.729+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:08.733 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.729+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:08.734 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.729+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:08.810 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.806+0000 7fd67aee6640 -1 reset not still connected to 0x55daafea6000 2026-02-20T21:27:08.812 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.808+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2d2e8f0 2026-02-20T21:27:08.812 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.808+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fde4e0 2026-02-20T21:27:08.812 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.808+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441450 2026-02-20T21:27:08.812 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.808+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:27:08.814 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.810+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d790 2026-02-20T21:27:08.814 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.810+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:27:08.814 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.810+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:27:08.814 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.810+0000 7fd67aee6640 -1 reset not still connected to 0x55dae421fd40 2026-02-20T21:27:08.814 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:08.810+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:27:09.181 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.177+0000 7fd67aee6640 -1 reset not still connected to 0x55daafea6000 2026-02-20T21:27:09.184 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.180+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423040 2026-02-20T21:27:09.185 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.180+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2d2e8f0 2026-02-20T21:27:09.185 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.181+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fde4e0 2026-02-20T21:27:09.185 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.181+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441450 2026-02-20T21:27:09.185 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.181+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:27:09.187 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.183+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d790 2026-02-20T21:27:09.187 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.183+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:27:09.187 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.183+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:27:09.187 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.183+0000 7fd67aee6640 -1 reset not still connected to 0x55dae421fd40 2026-02-20T21:27:09.187 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.183+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:27:09.194 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:09.190+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f6820 2026-02-20T21:27:09.194 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:09.191+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351e8f0 2026-02-20T21:27:09.195 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:09.191+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3555d40 2026-02-20T21:27:09.196 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:09.192+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:27:09.196 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:09.192+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0410 2026-02-20T21:27:09.197 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:09.192+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:27:09.197 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:09.192+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:27:09.235 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.231+0000 7fd67aee6640 -1 reset not still connected to 0x55daafea6000 2026-02-20T21:27:09.235 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.231+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2d2e8f0 2026-02-20T21:27:09.237 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.233+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fde4e0 2026-02-20T21:27:09.238 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.233+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441450 2026-02-20T21:27:09.238 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.233+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:27:09.239 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.235+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d790 2026-02-20T21:27:09.239 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.235+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:27:09.239 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.235+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:27:09.240 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.235+0000 7fd67aee6640 -1 reset not still connected to 0x55dae421fd40 2026-02-20T21:27:09.240 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.235+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:27:09.524 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:09.520+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:10.316 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:10.312+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92bc70 2026-02-20T21:27:10.503 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:10.499+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:27:10.503 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:10.499+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:27:10.505 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:10.501+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:27:10.505 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:10.501+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:27:10.505 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:10.501+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:27:10.505 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:10.501+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:27:10.505 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:10.501+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:27:10.505 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:10.501+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:27:10.506 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:10.501+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:27:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-20T21:27:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'image health:' 2026-02-20T21:27:10.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 3 2026-02-20T21:27:10.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T21:27:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=OK 2026-02-20T21:27:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test OK = ERROR 2026-02-20T21:27:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test OK = OK 2026-02-20T21:27:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: compare many images' 2026-02-20T21:27:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.IQNOxgiLiE/rbd-mirror.test.log 2026-02-20T21:27:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-20T21:27:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-20 21:27:10 TEST: compare many images 2026-02-20T21:27:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-20 21:27:10 TEST: compare many images 2026-02-20T21:27:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 50 2026-02-20T21:27:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:27:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_1 2026-02-20T21:27:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_1 snap 2026-02-20T21:27:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_1@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:27:11.052 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.048+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:11.053 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.049+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:27:11.055 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.051+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:11.055 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.051+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:11.055 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.051+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:11.055 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.051+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:11.055 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.051+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:11.055 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.051+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:11.055 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.051+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:11.056 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.051+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:11.425 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.421+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:27:11.427 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.423+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:27:11.427 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.423+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:27:11.428 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.423+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:27:11.428 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.423+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:27:11.428 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:11.423+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:11.593 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.589+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:27:11.595 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.591+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:27:11.595 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.592+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:27:11.595 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.592+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:27:11.596 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.592+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:27:11.718 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.714+0000 7fc9d03ed640 -1 reset not still connected to 0x5576df375790 2026-02-20T21:27:11.720 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.716+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e014d790 2026-02-20T21:27:11.720 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.716+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e14abc70 2026-02-20T21:27:11.720 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.716+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:27:11.721 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.717+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e21464e0 2026-02-20T21:27:11.721 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.717+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:27:11.721 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.717+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:27:11.722 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.717+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:27:11.722 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.717+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:27:11.722 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:11.717+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:27:12.282 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:12.278+0000 7fd67aee6640 -1 reset not still connected to 0x55daaf7e6dd0 2026-02-20T21:27:12.282 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:12.278+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441450 2026-02-20T21:27:12.283 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:12.279+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:27:12.773 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:12.769+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:12.774 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:12.771+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:27:12.776 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:12.772+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:27:12.776 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:12.772+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:12.776 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:12.772+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:27:12.776 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:12.772+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:12.777 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:12.774+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:12.777 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:12.774+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:12.778 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:12.774+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:12.833 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:12.829+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:27:12.833 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:12.829+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:27:12.835 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:12.831+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:27:12.835 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:12.831+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:27:12.835 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:12.831+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:27:12.992 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:12.988+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:27:13.013 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:13.009+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:13.013 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:13.009+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:13.030 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:13.027+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:27:13.031 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:13.027+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e21464e0 2026-02-20T21:27:13.031 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:13.027+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:27:13.031 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:13.027+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:27:13.031 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:13.027+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:27:13.031 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:13.027+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:27:14.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_1 2026-02-20T21:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_1 started 2026-02-20T21:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:27:14.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:27:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:27:14.690 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:14.686+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:27:14.690 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:14.686+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:27:14.690 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:14.686+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:27:14.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:14.716+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:14.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:14.716+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:14.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:14.716+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:14.720 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:14.716+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_1 started 2026-02-20T21:27:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:27:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:27:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:27:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_1 2026-02-20T21:27:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:27:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:27:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:27:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:27:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:27:15.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:15.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:27:15.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:15.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_1 2026-02-20T21:27:15.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:15.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:15.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:27:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:15.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:27:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:27:15.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:15.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:15.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:15.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:15.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_1 2026-02-20T21:27:15.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:15.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_15 2026-02-20T21:27:15.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_19 2026-02-20T21:27:15.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_1 2026-02-20T21:27:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:27:15.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:15.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:27:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:27:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:15.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:27:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:27:15.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:15.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_1 2026-02-20T21:27:15.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 4 closest matches: 2026-02-20T21:27:15.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_11 2026-02-20T21:27:15.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_13 2026-02-20T21:27:15.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_14 2026-02-20T21:27:15.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_16 2026-02-20T21:27:15.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_1 2026-02-20T21:27:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:27:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:15.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:15.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:27:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:27:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:27:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:15.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_1 2026-02-20T21:27:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:27:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:27:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:27:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:27:15.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:27:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:27:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:27:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_1 2026-02-20T21:27:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:27:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:27:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:27:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:15.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:27:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_1 2026-02-20T21:27:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:27:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:27:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:27:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:27:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:27:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:27:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:15.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:27:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_1 2026-02-20T21:27:15.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_1 ']' 2026-02-20T21:27:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-20T21:27:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-20T21:27:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:15.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-20T21:27:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:15.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:15.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:15.442 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:15.438+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:27:15.442 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:15.438+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:27:15.442 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:15.438+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:27:15.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:15.440+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:27:15.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:15.440+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:15.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:15.440+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:15.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:15.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:15.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_15 2026-02-20T21:27:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_19 2026-02-20T21:27:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:15.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:15.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2026-02-20T21:27:15.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:15.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:15.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:15.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:15.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:15.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:15.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:15.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:15.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:15.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:15.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:15.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 4 closest matches: 2026-02-20T21:27:15.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_11 2026-02-20T21:27:15.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_13 2026-02-20T21:27:15.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_14 2026-02-20T21:27:15.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_16 2026-02-20T21:27:15.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:15.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:15.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_1 2026-02-20T21:27:15.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:15.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:15.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:15.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:15.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:15.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:15.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:15.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:15.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:15.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:15.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:15.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:15.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:15.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:15.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:15.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:15.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:15.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:15.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:15.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:15.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-20T21:27:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-20T21:27:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:15.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-20T21:27:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-20T21:27:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:15.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-20T21:27:15.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:15.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:15.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310]]], state=connected] 2026-02-20T21:27:15.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310], [object_number=53, tag_tid=1, entry_tid=57309], [object_number=52, tag_tid=1, entry_tid=57308]]], state=connected] 2026-02-20T21:27:15.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:15.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:15.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-20T21:27:15.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:15.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:15.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:15.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-20T21:27:15.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:15.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:15.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:15.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:15.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-20T21:27:15.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-20T21:27:15.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:15.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:15.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-20T21:27:15.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:15.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:15.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310]]], state=connected] 2026-02-20T21:27:15.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310], [object_number=53, tag_tid=1, entry_tid=57309], [object_number=52, tag_tid=1, entry_tid=57308]]], state=connected] 2026-02-20T21:27:15.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:15.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:15.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:15.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=55, tag_tid=1, entry_tid=57311' '!=' '' 2026-02-20T21:27:15.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:27:15.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=55, tag_tid=1, entry_tid=57311' = '' 2026-02-20T21:27:15.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:15.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:15.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:27:15.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:27:15.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:27:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:15.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:27:15.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:15.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:27:15.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:15.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:27:15.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:15.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:27:15.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:15.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=57311 2026-02-20T21:27:15.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 57311 2026-02-20T21:27:15.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:27:15.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:15.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:27:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_1 2026-02-20T21:27:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_1 ']' 2026-02-20T21:27:15.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-20T21:27:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-20T21:27:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:15.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-20T21:27:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:15.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:15.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:15.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:15.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_15 2026-02-20T21:27:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_19 2026-02-20T21:27:15.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2026-02-20T21:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:15.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:15.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:15.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:15.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:15.965 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.961+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:27:15.966 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.961+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:27:15.966 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.961+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:27:15.966 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.961+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:15.966 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.961+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:27:15.966 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.961+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:15.966 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.961+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:15.966 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.961+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:15.966 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.961+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:15.966 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.961+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:15.967 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.962+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:15.967 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.962+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:15.967 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:15.963+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 4 closest matches: 2026-02-20T21:27:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_11 2026-02-20T21:27:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_13 2026-02-20T21:27:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_14 2026-02-20T21:27:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_16 2026-02-20T21:27:16.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_1 2026-02-20T21:27:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:16.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:16.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:16.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:16.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:16.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:16.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:16.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-20T21:27:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-20T21:27:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-20T21:27:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-20T21:27:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-20T21:27:16.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:16.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:16.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:16.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:16.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310]]], state=connected] 2026-02-20T21:27:16.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310], [object_number=53, tag_tid=1, entry_tid=57309], [object_number=52, tag_tid=1, entry_tid=57308]]], state=connected] 2026-02-20T21:27:16.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:16.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-20T21:27:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-20T21:27:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-20T21:27:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-20T21:27:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-20T21:27:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310]]], state=connected] 2026-02-20T21:27:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310], [object_number=53, tag_tid=1, entry_tid=57309], [object_number=52, tag_tid=1, entry_tid=57308]]], state=connected] 2026-02-20T21:27:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=55, tag_tid=1, entry_tid=57311' '!=' 'object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:16.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:27:16.143 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:16.139+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2d2e8f0 2026-02-20T21:27:16.144 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:16.140+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fde4e0 2026-02-20T21:27:16.144 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:16.140+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441450 2026-02-20T21:27:16.144 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:16.140+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:27:16.145 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:16.141+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d790 2026-02-20T21:27:16.145 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:16.141+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:27:16.145 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:16.141+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:27:16.145 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:16.141+0000 7fd67aee6640 -1 reset not still connected to 0x55dae421fd40 2026-02-20T21:27:16.145 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:16.141+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:27:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_1 2026-02-20T21:27:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_1 ']' 2026-02-20T21:27:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-20T21:27:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-20T21:27:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-20T21:27:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:16.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:16.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:16.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:16.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:16.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:16.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:16.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:16.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:16.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_15 2026-02-20T21:27:16.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_19 2026-02-20T21:27:16.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2026-02-20T21:27:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:16.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:16.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:16.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:16.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 4 closest matches: 2026-02-20T21:27:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_11 2026-02-20T21:27:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_13 2026-02-20T21:27:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_14 2026-02-20T21:27:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_16 2026-02-20T21:27:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:16.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_1 2026-02-20T21:27:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:16.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:16.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:16.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:16.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:16.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:16.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:16.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:16.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:16.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:16.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:16.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:16.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:16.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:16.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:16.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-20T21:27:16.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-20T21:27:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-20T21:27:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-20T21:27:16.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-20T21:27:16.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:16.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:16.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:16.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:16.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310]]], state=connected] 2026-02-20T21:27:16.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310], [object_number=53, tag_tid=1, entry_tid=57309], [object_number=52, tag_tid=1, entry_tid=57308]]], state=connected] 2026-02-20T21:27:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-20T21:27:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-20T21:27:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-20T21:27:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-20T21:27:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-20T21:27:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:16.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:16.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:16.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:16.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310]]], state=connected] 2026-02-20T21:27:16.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310], [object_number=53, tag_tid=1, entry_tid=57309], [object_number=52, tag_tid=1, entry_tid=57308]]], state=connected] 2026-02-20T21:27:16.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:16.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:16.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:16.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=55, tag_tid=1, entry_tid=57311' '!=' 'object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:16.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:16.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:27:16.889 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:16.885+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:27:16.889 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:16.885+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:27:16.891 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:16.887+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:27:16.892 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:16.888+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:27:16.892 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:16.888+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:27:16.892 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:16.888+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:27:16.892 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:16.888+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:27:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_1 2026-02-20T21:27:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_1 ']' 2026-02-20T21:27:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-20T21:27:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-20T21:27:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:17.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:17.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:17.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-20T21:27:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:17.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:17.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:17.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:17.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_15 2026-02-20T21:27:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_19 2026-02-20T21:27:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2026-02-20T21:27:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:17.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:17.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:17.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:17.675+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:27:17.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:17.675+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:17.679 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:17.675+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 4 closest matches: 2026-02-20T21:27:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_11 2026-02-20T21:27:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_13 2026-02-20T21:27:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_14 2026-02-20T21:27:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_16 2026-02-20T21:27:17.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_1 2026-02-20T21:27:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:17.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:17.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:17.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:17.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-20T21:27:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-20T21:27:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-20T21:27:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-20T21:27:17.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:17.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:17.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-20T21:27:17.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:17.789 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:17.785+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:27:17.789 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:17.785+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:27:17.791 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:17.786+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:27:17.791 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:17.786+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:27:17.791 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:17.787+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:27:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310]]], state=connected] 2026-02-20T21:27:17.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310], [object_number=53, tag_tid=1, entry_tid=57309], [object_number=52, tag_tid=1, entry_tid=57308]]], state=connected] 2026-02-20T21:27:17.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:17.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-20T21:27:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:17.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-20T21:27:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-20T21:27:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-20T21:27:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-20T21:27:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310]]], state=connected] 2026-02-20T21:27:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310], [object_number=53, tag_tid=1, entry_tid=57309], [object_number=52, tag_tid=1, entry_tid=57308]]], state=connected] 2026-02-20T21:27:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=55, tag_tid=1, entry_tid=57311' '!=' 'object_number=55, tag_tid=1, entry_tid=57311' 2026-02-20T21:27:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:17.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:27:18.032 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:18.028+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:18.032 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:18.028+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:18.032 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:18.028+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:18.032 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:18.028+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:18.032 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:18.028+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:18.032 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:18.028+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:18.701 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:18.697+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:27:18.702 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:18.697+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:19.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_1 2026-02-20T21:27:19.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_1 ']' 2026-02-20T21:27:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_1' 2026-02-20T21:27:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_1 2026-02-20T21:27:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:19.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_1 2026-02-20T21:27:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:19.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:19.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:19.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:19.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:19.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:19.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:19.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:19.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:19.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_15 2026-02-20T21:27:19.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_19 2026-02-20T21:27:19.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_1 2026-02-20T21:27:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:19.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:19.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:19.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 4 closest matches: 2026-02-20T21:27:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_11 2026-02-20T21:27:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_13 2026-02-20T21:27:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_14 2026-02-20T21:27:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_16 2026-02-20T21:27:19.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:19.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:19.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_1 2026-02-20T21:27:19.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:19.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:19.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:19.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:19.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:19.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:19.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_1 2026-02-20T21:27:19.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_1 2026-02-20T21:27:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '' 2026-02-20T21:27:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:19.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-20T21:27:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-20T21:27:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:19.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-20T21:27:19.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:19.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310]]], state=connected] 2026-02-20T21:27:19.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310]]], state=connected] 2026-02-20T21:27:19.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:19.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:19.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_1 2026-02-20T21:27:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_1 '..*' 2026-02-20T21:27:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:27:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_1.status 2026-02-20T21:27:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_1.status 2026-02-20T21:27:19.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_1 2026-02-20T21:27:19.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:19.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310]]], state=connected] 2026-02-20T21:27:19.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=55, tag_tid=1, entry_tid=57311], [object_number=54, tag_tid=1, entry_tid=57310]]], state=connected] 2026-02-20T21:27:19.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.status 2026-02-20T21:27:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:19.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_1 snap 2026-02-20T21:27:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:27:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:27:19.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:27:20.684 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:20.680+0000 7fd67aee6640 -1 reset not still connected to 0x55dae421fd40 2026-02-20T21:27:20.684 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:20.680+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:27:20.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_1@snap 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_1': 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ff32e5a524 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ff32e5a524 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:18:49 2026 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:18:49 2026 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:19:50 2026 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ff32e5a524 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:27:20.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 5fc907af-d359-43e4-a566-9a99c2e495fe 2026-02-20T21:27:20.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:27:20.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:20.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_1 snap 2026-02-20T21:27:20.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:20.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:27:20.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:20.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:27:20.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.export 2026-02-20T21:27:20.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_1.export 2026-02-20T21:27:20.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_1.export 2026-02-20T21:27:20.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_1@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.export 2026-02-20T21:27:21.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:27:21.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_1@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_1.export 2026-02-20T21:27:21.069 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:21.064+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:27:21.069 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:21.064+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:27:21.069 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:21.064+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:27:21.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:27:21.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_1.export 2026-02-20T21:27:21.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_1.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_1.export 2026-02-20T21:27:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:27:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_2 2026-02-20T21:27:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_2 snap 2026-02-20T21:27:21.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:21.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_2@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:27:21.703 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:21.698+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:27:21.703 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:21.698+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:27:21.703 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:21.698+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:27:21.714 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:21.710+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:21.714 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:21.710+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:21.717 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:21.713+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:22.153 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.149+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:27:22.154 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.150+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d790 2026-02-20T21:27:22.155 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.150+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:27:22.155 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.150+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:27:22.156 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.152+0000 7fd67aee6640 -1 reset not still connected to 0x55dae421fd40 2026-02-20T21:27:22.156 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.152+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:27:22.174 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:22.170+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:27:22.175 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:22.170+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:27:22.175 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:22.170+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:27:22.175 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:22.171+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:27:22.175 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:22.171+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:27:22.177 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:22.172+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:27:22.178 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:22.174+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e16c0 2026-02-20T21:27:22.178 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:22.174+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:27:22.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef380 2026-02-20T21:27:22.224 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.220+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:22.225 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.222+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:22.228 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.223+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:22.228 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.223+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:22.228 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.223+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:22.619 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.615+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:22.620 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.616+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:22.621 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.617+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:22.622 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.618+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:22.622 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.618+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:22.622 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.618+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:22.622 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.618+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:22.623 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.619+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:22.884 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.880+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:22.886 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.881+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:22.886 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.881+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:22.886 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.881+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:22.886 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.881+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:22.886 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:22.881+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:22.953 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:22.949+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:23.634 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.630+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:27:23.635 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.631+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:27:23.636 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.631+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:23.636 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.631+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:27:23.637 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.633+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:23.637 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.633+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:23.637 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.633+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:23.637 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.633+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:23.637 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.633+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:23.638 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.633+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:23.638 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.633+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:23.638 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.633+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:23.919 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.915+0000 7f413d8ae640 -1 reset not still connected to 0x55d26d997c70 2026-02-20T21:27:23.920 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.915+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:27:23.920 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.916+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:27:23.920 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.916+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:27:23.920 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.916+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:27:23.921 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.918+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107a1a0 2026-02-20T21:27:23.921 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.918+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:27:23.922 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.918+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:27:23.922 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.918+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:27:23.922 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.918+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:23.955 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.951+0000 7f413d8ae640 -1 reset not still connected to 0x55d26d997c70 2026-02-20T21:27:23.955 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.951+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:27:23.955 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.951+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:27:23.956 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.951+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:27:23.957 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.954+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:27:23.958 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.954+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:27:23.958 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:23.954+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_2 2026-02-20T21:27:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_2 started 2026-02-20T21:27:24.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:24.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:24.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:24.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:27:24.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:27:24.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:24.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:27:24.180 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:24.175+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:24.445 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.441+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:27:24.445 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.441+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:24.446 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.442+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:27:24.447 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.443+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:24.447 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.443+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:24.447 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.443+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:24.447 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.443+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:24.448 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.443+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:24.448 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.444+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:24.448 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.444+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:24.448 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.444+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:24.538 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.534+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b8dd0 2026-02-20T21:27:24.538 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.534+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9110 2026-02-20T21:27:24.541 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.537+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:27:24.572 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.568+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b8dd0 2026-02-20T21:27:24.572 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.568+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9110 2026-02-20T21:27:24.572 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.568+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:27:24.573 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:24.568+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:27:24.709 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:24.705+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:27:24.710 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:24.705+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:27:24.710 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:24.705+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:27:24.999 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:24.995+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e31435f0 2026-02-20T21:27:25.001 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:24.996+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:27:25.001 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:24.996+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:27:25.002 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:24.997+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:27:25.002 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:24.997+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:27:25.002 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:24.997+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:27:25.002 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:24.998+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:27:25.003 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:24.999+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e16c0 2026-02-20T21:27:25.003 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:24.999+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:27:25.043 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.039+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:27:25.044 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.039+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:27:25.044 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.039+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:27:25.045 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.041+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:27:25.045 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.041+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:27:25.045 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.041+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:27:25.046 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.041+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:27:25.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_2 started 2026-02-20T21:27:25.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:25.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:25.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:27:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:27:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:27:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_2 2026-02-20T21:27:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:27:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:27:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:25.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:27:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:27:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:27:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:25.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:27:25.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_2 2026-02-20T21:27:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:27:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:27:25.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:27:25.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:25.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_2 2026-02-20T21:27:25.112 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:25.108+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:25.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:25.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_21 2026-02-20T21:27:25.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_23 2026-02-20T21:27:25.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_24 2026-02-20T21:27:25.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:25.115 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:25.110+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:27:25.115 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:25.110+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:25.115 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:25.110+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:27:25.116 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:25.110+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:25.116 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:25.110+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:25.116 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:25.110+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:25.116 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:25.110+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:25.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:25.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_2 2026-02-20T21:27:25.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:27:25.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:25.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:25.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:25.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:25.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:25.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:25.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:25.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:27:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:27:25.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:25.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:25.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:25.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:25.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_2 2026-02-20T21:27:25.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:25.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_20 2026-02-20T21:27:25.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_27 2026-02-20T21:27:25.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_2 2026-02-20T21:27:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:27:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:27:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:27:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:27:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:27:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:25.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:25.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:25.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:25.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_2 2026-02-20T21:27:25.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:27:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:27:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:27:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:27:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:27:25.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:27:25.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:27:25.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_2 2026-02-20T21:27:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:27:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:27:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:27:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:27:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_2 2026-02-20T21:27:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:27:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:25.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:27:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:27:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:27:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:27:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:27:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:25.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:27:25.376 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.372+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:27:25.377 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.374+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:27:25.378 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.374+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:27:25.378 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.374+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07ade10 2026-02-20T21:27:25.378 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.374+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:27:25.379 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.375+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:27:25.379 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:25.375+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:27:25.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_2 2026-02-20T21:27:25.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:25.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:25.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:25.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:25.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_2 ']' 2026-02-20T21:27:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-20T21:27:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-20T21:27:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:25.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-20T21:27:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:25.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:25.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:25.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:25.450 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:25.441+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92bc70 2026-02-20T21:27:25.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:25.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_21 2026-02-20T21:27:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_23 2026-02-20T21:27:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_24 2026-02-20T21:27:25.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-20T21:27:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:25.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:25.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:25.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:25.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:25.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_20 2026-02-20T21:27:25.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_27 2026-02-20T21:27:25.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:25.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:25.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-20T21:27:25.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:25.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:25.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:25.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:25.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:25.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:25.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:25.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:25.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-20T21:27:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-20T21:27:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-20T21:27:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-20T21:27:25.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:25.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:25.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-20T21:27:25.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:25.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:27:25.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:27:25.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:25.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31978], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:25.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31978], [object_number=29, tag_tid=1, entry_tid=31977], [object_number=28, tag_tid=1, entry_tid=31976], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:25.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:25.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:27:25.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-20T21:27:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-20T21:27:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-20T21:27:25.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-20T21:27:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-20T21:27:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:25.664 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:25.660+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:27:25.665 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:25.660+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:25.668 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:25.665+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:25.669 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:25.665+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:25.669 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:25.665+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:25.670 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:25.665+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:25.670 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:25.665+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:25.670 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:25.665+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:25.670 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:25.665+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:25.670 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:25.665+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:27:25.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:27:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31978], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:25.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31978], [object_number=29, tag_tid=1, entry_tid=31977], [object_number=28, tag_tid=1, entry_tid=31976], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:25.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:25.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:25.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:25.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=31978' '!=' '' 2026-02-20T21:27:25.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:27:25.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=31978' = '' 2026-02-20T21:27:25.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:25.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:27:25.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:27:25.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:27:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:27:25.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:25.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:27:25.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:27:25.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:27:25.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:27:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:27:25.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31978 2026-02-20T21:27:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 31978 2026-02-20T21:27:25.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:27:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:25.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:27:25.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_2 2026-02-20T21:27:25.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:25.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:25.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:25.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_2 ']' 2026-02-20T21:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-20T21:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-20T21:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:25.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-20T21:27:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:25.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:25.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:25.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:25.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:25.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:25.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_21 2026-02-20T21:27:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_23 2026-02-20T21:27:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_24 2026-02-20T21:27:25.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:25.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:25.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-20T21:27:25.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:25.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:25.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:25.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:25.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:25.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:25.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:25.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:25.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:26.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_20 2026-02-20T21:27:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_27 2026-02-20T21:27:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-20T21:27:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:26.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:26.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:26.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:26.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:26.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:26.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-20T21:27:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-20T21:27:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-20T21:27:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-20T21:27:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-20T21:27:26.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:27:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:27:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31978], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31978], [object_number=29, tag_tid=1, entry_tid=31977], [object_number=28, tag_tid=1, entry_tid=31976], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:27:26.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-20T21:27:26.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:26.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:26.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:26.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-20T21:27:26.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:26.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:26.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:26.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:26.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-20T21:27:26.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-20T21:27:26.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:26.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:26.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-20T21:27:26.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:26.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:27:26.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:27:26.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:26.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31978], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:26.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31978], [object_number=29, tag_tid=1, entry_tid=31977], [object_number=28, tag_tid=1, entry_tid=31976], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:26.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:26.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:26.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=31978' '!=' 'object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:27:26.224 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.220+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1724e0 2026-02-20T21:27:26.224 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.220+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:27:26.224 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.220+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:27:26.224 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.220+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:27:26.224 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.220+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:26.226 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.222+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:26.226 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.222+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:26.226 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.222+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:26.226 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.222+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:26.227 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.222+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:26.227 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.223+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:26.230 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.226+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:26.413 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.409+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:27:26.415 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.411+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:27:26.415 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.411+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:27:26.415 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.411+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107a1a0 2026-02-20T21:27:26.415 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.411+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:27:26.415 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.411+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:27:26.415 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.411+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:27:26.415 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.411+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:26.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_2 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_2 ']' 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:26.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:26.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:26.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:26.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:26.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_21 2026-02-20T21:27:26.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_23 2026-02-20T21:27:26.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_24 2026-02-20T21:27:26.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:26.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:26.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:26.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:26.643 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:26.639+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_20 2026-02-20T21:27:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_27 2026-02-20T21:27:26.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:26.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:26.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:26.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:26.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:26.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:26.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:26.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-20T21:27:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-20T21:27:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-20T21:27:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-20T21:27:26.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:26.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:26.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-20T21:27:26.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:26.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:27:26.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:27:26.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:26.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31978], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:26.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31978], [object_number=29, tag_tid=1, entry_tid=31977], [object_number=28, tag_tid=1, entry_tid=31976], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:27:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-20T21:27:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-20T21:27:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:26.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-20T21:27:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-20T21:27:26.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:26.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:26.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-20T21:27:26.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:27:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:27:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31978], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:26.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31978], [object_number=29, tag_tid=1, entry_tid=31977], [object_number=28, tag_tid=1, entry_tid=31976], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:26.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=31978' '!=' 'object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:26.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:27:26.979 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:26.975+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92bc70 2026-02-20T21:27:27.140 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.136+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:27:27.140 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.137+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef380 2026-02-20T21:27:27.141 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.137+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:27.143 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.139+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:27:27.143 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.139+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:27:27.144 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.140+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:27.144 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.140+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:27:27.144 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.140+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:27.144 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.140+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:27.144 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.140+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:27.144 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.140+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:27.299 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.295+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:27:27.302 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.298+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef380 2026-02-20T21:27:27.302 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.298+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:27.302 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.298+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:27:27.302 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.298+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:27:27.302 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.299+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:27.303 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.299+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:27:27.305 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.302+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:27.305 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.302+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:27.306 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.302+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:27.306 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.302+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:27.507 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:27.504+0000 7f413d8ae640 -1 reset not still connected to 0x55d26e400f70 2026-02-20T21:27:27.508 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:27.504+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1724e0 2026-02-20T21:27:27.509 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:27.505+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:27:27.509 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:27.505+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:27:27.509 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:27.505+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:27.509 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:27.505+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:27:27.510 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:27.507+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:27.510 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:27.507+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:27.510 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:27.507+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:27.510 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:27.507+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:27.510 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:27.507+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:27.511 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:27.507+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_2 2026-02-20T21:27:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_2 ']' 2026-02-20T21:27:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-20T21:27:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-20T21:27:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:27.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:27.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:27.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:27.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:27.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:27.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:27.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-20T21:27:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:27.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:27.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_21 2026-02-20T21:27:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_23 2026-02-20T21:27:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_24 2026-02-20T21:27:27.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:27.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:27.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:27.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:27.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_20 2026-02-20T21:27:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_27 2026-02-20T21:27:27.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:27.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:27.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:27.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:27.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:27.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-20T21:27:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:27.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-20T21:27:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-20T21:27:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-20T21:27:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-20T21:27:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:27.966 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.962+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:27:27.967 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.963+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:27:27.967 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.963+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:27:27.967 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.963+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:27:27.967 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:27.963+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:27:28.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:27:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:27:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31978], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31978], [object_number=29, tag_tid=1, entry_tid=31977], [object_number=28, tag_tid=1, entry_tid=31976], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:28.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:28.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:27:28.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-20T21:27:28.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:28.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:28.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:28.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-20T21:27:28.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:28.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:28.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:28.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:28.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-20T21:27:28.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-20T21:27:28.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:28.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:28.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-20T21:27:28.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:27:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:27:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31978], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=31978], [object_number=29, tag_tid=1, entry_tid=31977], [object_number=28, tag_tid=1, entry_tid=31976], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:28.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:28.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:28.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=31978' '!=' 'object_number=30, tag_tid=1, entry_tid=31978' 2026-02-20T21:27:28.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:28.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:27:28.122 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:28.118+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:27:28.124 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:28.120+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:27:28.124 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:28.120+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:27:28.124 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:28.120+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:27:28.125 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:28.120+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:28.529 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:28.525+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9110 2026-02-20T21:27:28.529 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:28.525+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:27:28.529 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:28.525+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:27:29.650 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:29.646+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:27:29.651 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:29.647+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:27:29.651 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:29.647+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:27:29.651 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:29.647+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:27:29.652 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:29.647+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:27:29.652 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:29.647+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:27:29.652 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:29.647+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:27:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_2 2026-02-20T21:27:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:29.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_2 ']' 2026-02-20T21:27:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_2' 2026-02-20T21:27:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_2 2026-02-20T21:27:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:29.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:29.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:29.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:29.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:29.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:29.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:29.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:29.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_2 2026-02-20T21:27:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:29.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:29.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:29.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:29.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:29.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_21 2026-02-20T21:27:29.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_23 2026-02-20T21:27:29.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_24 2026-02-20T21:27:29.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:29.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:29.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_2 2026-02-20T21:27:29.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:29.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:29.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:29.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:29.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:29.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:29.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:29.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:29.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:29.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:29.758 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:29.754+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:27:29.758 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:29.754+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:27:29.758 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:29.754+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:27:29.758 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:29.754+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:29.758 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:29.754+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:29.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:29.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_20 2026-02-20T21:27:29.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_27 2026-02-20T21:27:29.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_2 2026-02-20T21:27:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:29.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:29.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:29.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:29.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:29.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:29.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:29.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_2 2026-02-20T21:27:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:29.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_2 2026-02-20T21:27:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '' 2026-02-20T21:27:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:29.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:29.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-20T21:27:29.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-20T21:27:29.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:29.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:29.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:29.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-20T21:27:29.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:27:29.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:27:29.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:29.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31978], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:29.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31978], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:29.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:29.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:27:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_2 2026-02-20T21:27:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:29.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:29.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_2 '..*' 2026-02-20T21:27:29.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:29.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:29.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:27:29.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:29.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_2.status 2026-02-20T21:27:29.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_2.status 2026-02-20T21:27:29.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:29.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:29.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_2 2026-02-20T21:27:29.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:27:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:27:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31978], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:29.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=31978], [object_number=31, tag_tid=1, entry_tid=31975]]], state=connected] 2026-02-20T21:27:29.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.status 2026-02-20T21:27:29.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:27:29.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:27:29.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:29.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_2 snap 2026-02-20T21:27:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:27:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:27:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:27:30.120 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.116+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:27:30.121 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.116+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:27:30.122 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.118+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:27:30.123 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.119+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:27:30.123 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.119+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:27:30.123 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.119+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:30.176 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.172+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:30.177 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.173+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:30.178 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.173+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:30.178 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.173+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:30.178 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.173+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:30.178 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.173+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:30.317 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.313+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:27:30.317 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.313+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:27:30.318 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.313+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:27:30.318 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.313+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:27:30.318 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.313+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:27:30.318 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.313+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:27:30.318 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.314+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:27:30.319 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.314+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:27:30.412 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.408+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:30.413 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.409+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:30.413 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.409+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:30.414 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.409+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:30.414 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.409+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:30.414 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.409+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:30.414 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:30.410+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:30.481 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:30.476+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e14abc70 2026-02-20T21:27:30.481 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:30.476+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:27:30.481 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:30.476+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:27:30.481 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:30.476+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:27:30.481 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:30.476+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:27:30.481 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:30.476+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:27:30.481 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:30.476+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:27:30.566 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.562+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:27:30.567 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.564+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:27:30.567 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.564+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef380 2026-02-20T21:27:30.568 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.564+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:30.569 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.565+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:27:30.570 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.565+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:27:30.570 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.565+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:30.570 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.565+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:27:30.570 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.565+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:30.570 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.565+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:30.570 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:30.565+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:30.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_2@snap 2026-02-20T21:27:30.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_2': 2026-02-20T21:27:30.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:27:30.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:27:30.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:27:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ec6815b18 2026-02-20T21:27:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ec6815b18 2026-02-20T21:27:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:27:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:27:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:27:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:27:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:19:03 2026 2026-02-20T21:27:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:19:03 2026 2026-02-20T21:27:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:19:03 2026 2026-02-20T21:27:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:27:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ec6815b18 2026-02-20T21:27:30.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:27:30.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:27:30.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 04f670ee-1f66-4b5c-95e3-3b7bb0b9ac74 2026-02-20T21:27:30.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:27:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_2 snap 2026-02-20T21:27:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:27:30.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:27:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.export 2026-02-20T21:27:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_2.export 2026-02-20T21:27:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_2.export 2026-02-20T21:27:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_2@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.export 2026-02-20T21:27:31.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:27:31.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_2@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_2.export 2026-02-20T21:27:31.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:27:31.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_2.export 2026-02-20T21:27:31.262 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:31.258+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:27:31.262 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:31.258+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:27:31.262 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:31.258+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef380 2026-02-20T21:27:31.262 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:31.258+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:31.265 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:31.261+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:27:31.266 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:31.261+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:27:31.266 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:31.261+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:31.266 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:31.261+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:27:31.266 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:31.261+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:31.267 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:31.263+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:31.269 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:31.265+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:31.269 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:31.265+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:31.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_2.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_2.export 2026-02-20T21:27:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_3 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_3 snap 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:31.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_3@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:27:32.304 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:32.299+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:27:32.304 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:32.299+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:27:32.304 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:32.299+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:27:32.304 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:32.299+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:27:32.306 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:32.302+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:27:32.306 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:32.302+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:27:32.307 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:32.302+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:27:32.309 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:32.305+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:33.266 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:33.262+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9110 2026-02-20T21:27:33.266 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:33.262+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:27:33.266 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:33.262+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:27:33.543 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:33.539+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:27:33.543 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:33.539+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:27:33.543 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:33.539+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:27:33.543 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:33.539+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:27:33.543 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:33.539+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:33.543 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:33.539+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:34.032 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:34.028+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:27:34.032 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:34.028+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:27:34.032 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:34.028+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:27:34.034 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:34.031+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:34.035 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:34.031+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_3 2026-02-20T21:27:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_3 started 2026-02-20T21:27:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:27:34.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:27:34.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:34.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:27:34.301 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:34.297+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:27:34.302 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:34.298+0000 7fd67aee6640 -1 reset not still connected to 0x55dace5245b0 2026-02-20T21:27:34.302 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:34.298+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:27:34.304 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:34.300+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:27:34.304 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:34.300+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:27:34.304 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:34.300+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:27:34.304 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:34.300+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:27:34.425 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:34.420+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:27:34.425 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:34.420+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:34.425 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:34.421+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:27:34.426 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:34.422+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:34.426 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:34.422+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:34.427 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:34.424+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:34.428 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:34.424+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:34.428 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:34.424+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:34.428 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:34.424+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:34.428 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:34.424+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:34.428 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:34.424+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:35.060 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.056+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2147380 2026-02-20T21:27:35.061 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.056+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2588c30 2026-02-20T21:27:35.061 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.056+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351e8f0 2026-02-20T21:27:35.061 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.056+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3555d40 2026-02-20T21:27:35.061 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.056+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:27:35.062 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.058+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0410 2026-02-20T21:27:35.062 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.058+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:27:35.062 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.058+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:27:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_3 started 2026-02-20T21:27:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:35.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:27:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:27:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_3 2026-02-20T21:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:27:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:27:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:27:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:27:35.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_3 2026-02-20T21:27:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:35.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:27:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:27:35.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:35.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:35.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_3 2026-02-20T21:27:35.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 5 closest matches: 2026-02-20T21:27:35.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_30 2026-02-20T21:27:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_32 2026-02-20T21:27:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_34 2026-02-20T21:27:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_36 2026-02-20T21:27:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_38 2026-02-20T21:27:35.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:35.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_3 2026-02-20T21:27:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:27:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:27:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:27:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:27:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:35.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:35.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_3 2026-02-20T21:27:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_31 2026-02-20T21:27:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_35 2026-02-20T21:27:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_39 2026-02-20T21:27:35.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_3 2026-02-20T21:27:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:27:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:35.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:35.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:35.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:27:35.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:27:35.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:35.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_3 2026-02-20T21:27:35.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-20T21:27:35.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:35.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:35.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:35.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_3 2026-02-20T21:27:35.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:27:35.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:35.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:35.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:35.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:27:35.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:35.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:35.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:35.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:27:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:27:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:27:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:27:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:27:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:27:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:27:35.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:35.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_3 2026-02-20T21:27:35.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:27:35.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:27:35.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:27:35.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:27:35.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:27:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:27:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:27:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_3 2026-02-20T21:27:35.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:27:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:27:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:27:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:27:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_3 2026-02-20T21:27:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:27:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:27:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:27:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:27:35.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:27:35.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:27:35.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:35.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:27:35.460 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.456+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e014d790 2026-02-20T21:27:35.460 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.456+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e14abc70 2026-02-20T21:27:35.460 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.456+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:27:35.460 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.456+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e21464e0 2026-02-20T21:27:35.460 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.456+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:27:35.461 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.457+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:27:35.461 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.457+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:27:35.461 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.457+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:27:35.461 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:35.457+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:27:35.551 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:35.547+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:35.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-20T21:27:35.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:35.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:35.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:35.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:35.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-20T21:27:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-20T21:27:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-20T21:27:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:35.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:35.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-20T21:27:35.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:35.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:35.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:35.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:35.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:35.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:35.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:35.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:35.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:35.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:35.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:35.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:35.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:35.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:35.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:35.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:35.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:35.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:35.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 5 closest matches: 2026-02-20T21:27:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_30 2026-02-20T21:27:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_32 2026-02-20T21:27:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_34 2026-02-20T21:27:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_36 2026-02-20T21:27:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_38 2026-02-20T21:27:35.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-02-20T21:27:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:35.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:35.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:35.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:35.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:35.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:35.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:35.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:35.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:35.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_31 2026-02-20T21:27:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_35 2026-02-20T21:27:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_39 2026-02-20T21:27:35.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-02-20T21:27:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:35.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:35.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:35.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:35.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:35.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:35.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:35.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:35.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-20T21:27:35.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:27:35.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:35.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:35.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-02-20T21:27:35.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:27:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:27:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:27:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:35.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:27:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:27:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:27:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:35.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:27:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:27:35.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:35.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:35.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:35.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:35.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:35.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:35.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-20T21:27:35.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:35.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:35.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:35.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-20T21:27:35.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:35.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:35.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:35.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:35.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:27:35.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-20T21:27:35.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-20T21:27:35.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:35.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=56990], [object_number=53, tag_tid=1, entry_tid=56989], [object_number=52, tag_tid=1, entry_tid=56988], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:35.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:35.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-20T21:27:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-20T21:27:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:27:35.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-20T21:27:35.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:35.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:35.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-20T21:27:35.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:35.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:35.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:35.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:35.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:35.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=56990], [object_number=53, tag_tid=1, entry_tid=56989], [object_number=52, tag_tid=1, entry_tid=56988], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:35.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:35.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:35.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:35.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=56990' '!=' '' 2026-02-20T21:27:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:27:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=56990' = '' 2026-02-20T21:27:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:27:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:27:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:27:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:27:35.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:35.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:27:35.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:35.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:27:35.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:35.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:27:35.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:35.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=56990 2026-02-20T21:27:35.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 56990 2026-02-20T21:27:35.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:27:35.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:35.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:36.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:36.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:36.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:36.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:36.131 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:36.128+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441450 2026-02-20T21:27:36.132 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:36.128+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:27:36.134 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:36.130+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:27:36.135 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:36.131+0000 7fd67aee6640 -1 reset not still connected to 0x55dae421fd40 2026-02-20T21:27:36.135 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:36.131+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:27:36.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 5 closest matches: 2026-02-20T21:27:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_30 2026-02-20T21:27:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_32 2026-02-20T21:27:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_34 2026-02-20T21:27:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_36 2026-02-20T21:27:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_38 2026-02-20T21:27:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:36.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:36.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-02-20T21:27:36.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:36.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:36.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:36.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_31 2026-02-20T21:27:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_35 2026-02-20T21:27:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_39 2026-02-20T21:27:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:36.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:36.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-02-20T21:27:36.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:36.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:36.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:36.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:36.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:36.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-20T21:27:36.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:27:36.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-02-20T21:27:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:27:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:27:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:36.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:27:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:27:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:27:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:27:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:27:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:27:36.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:36.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-20T21:27:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-20T21:27:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:36.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:27:36.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-20T21:27:36.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:36.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-20T21:27:36.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:36.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:36.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:36.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:36.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:36.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=56990], [object_number=53, tag_tid=1, entry_tid=56989], [object_number=52, tag_tid=1, entry_tid=56988], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:36.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:36.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:36.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-20T21:27:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-20T21:27:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:27:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-20T21:27:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:36.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:36.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-20T21:27:36.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:36.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=56990], [object_number=53, tag_tid=1, entry_tid=56989], [object_number=52, tag_tid=1, entry_tid=56988], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:36.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:36.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:36.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:36.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=56990' '!=' 'object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:36.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:36.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:27:36.404 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:36.400+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:36.405 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:36.401+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:36.405 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:36.401+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:36.407 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:36.403+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:36.407 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:36.403+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:36.407 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:36.403+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:36.423 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.419+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1c06d00 2026-02-20T21:27:36.426 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.422+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2588c30 2026-02-20T21:27:36.426 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.423+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f6820 2026-02-20T21:27:36.427 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.423+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351e8f0 2026-02-20T21:27:36.427 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.423+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3555d40 2026-02-20T21:27:36.427 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.423+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:27:36.427 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.423+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0410 2026-02-20T21:27:36.427 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.423+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:27:36.428 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.423+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:27:36.785 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.781+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3555d40 2026-02-20T21:27:36.787 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.783+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:27:36.789 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.785+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0410 2026-02-20T21:27:36.789 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.785+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:27:36.789 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:36.785+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:27:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-20T21:27:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-20T21:27:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-20T21:27:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-20T21:27:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:36.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:36.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-20T21:27:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:36.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:36.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:36.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:36.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:36.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 5 closest matches: 2026-02-20T21:27:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_30 2026-02-20T21:27:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_32 2026-02-20T21:27:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_34 2026-02-20T21:27:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_36 2026-02-20T21:27:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_38 2026-02-20T21:27:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-02-20T21:27:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:36.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:36.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:36.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:36.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:36.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:36.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:36.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:36.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_31 2026-02-20T21:27:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_35 2026-02-20T21:27:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_39 2026-02-20T21:27:36.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-02-20T21:27:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:36.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:36.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:36.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:36.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:36.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-20T21:27:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:27:36.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:27:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:27:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:27:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:36.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:37.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-20T21:27:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:37.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-20T21:27:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:27:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-20T21:27:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:37.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-20T21:27:37.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:37.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=56990], [object_number=53, tag_tid=1, entry_tid=56989], [object_number=52, tag_tid=1, entry_tid=56988], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:37.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:37.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:37.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-20T21:27:37.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:37.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:37.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:37.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-20T21:27:37.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:37.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:37.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:37.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:37.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:27:37.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-20T21:27:37.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:37.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:37.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-20T21:27:37.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:37.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:37.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:37.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:37.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:37.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=56990], [object_number=53, tag_tid=1, entry_tid=56989], [object_number=52, tag_tid=1, entry_tid=56988], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:37.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=56990' '!=' 'object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:27:37.643 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:37.639+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9baea0 2026-02-20T21:27:37.644 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:37.639+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:27:37.644 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:37.639+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:27:37.644 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:37.640+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:27:37.644 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:37.640+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:37.645 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:37.640+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:37.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-20T21:27:37.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:37.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:37.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-20T21:27:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-20T21:27:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-20T21:27:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:37.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:37.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-20T21:27:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:37.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 5 closest matches: 2026-02-20T21:27:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_30 2026-02-20T21:27:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_32 2026-02-20T21:27:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_34 2026-02-20T21:27:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_36 2026-02-20T21:27:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_38 2026-02-20T21:27:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-02-20T21:27:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:37.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:37.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:37.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:37.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:37.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:37.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:37.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_31 2026-02-20T21:27:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_35 2026-02-20T21:27:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_39 2026-02-20T21:27:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-02-20T21:27:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:38.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:38.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:38.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:38.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-20T21:27:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:27:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-02-20T21:27:38.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:27:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:27:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:27:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:38.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:38.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:38.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:38.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:27:38.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:27:38.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:38.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-20T21:27:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-20T21:27:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:27:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-20T21:27:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-20T21:27:38.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:38.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:38.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:38.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:38.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=56990], [object_number=53, tag_tid=1, entry_tid=56989], [object_number=52, tag_tid=1, entry_tid=56988], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:38.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:38.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-20T21:27:38.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-20T21:27:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:27:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-20T21:27:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-20T21:27:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:38.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:38.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:38.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:38.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:38.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=56990], [object_number=53, tag_tid=1, entry_tid=56989], [object_number=52, tag_tid=1, entry_tid=56988], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:38.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:38.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=56990' '!=' 'object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:38.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:38.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:27:38.906 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:38.902+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b8dd0 2026-02-20T21:27:38.906 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:38.902+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9110 2026-02-20T21:27:38.907 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:38.903+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:27:38.982 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:38.978+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b25b0 2026-02-20T21:27:38.982 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:38.978+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92bc70 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:39.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:39.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:39.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:39.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:39.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 5 closest matches: 2026-02-20T21:27:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_30 2026-02-20T21:27:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_32 2026-02-20T21:27:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_34 2026-02-20T21:27:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_36 2026-02-20T21:27:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_38 2026-02-20T21:27:39.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:39.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:39.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:39.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_31 2026-02-20T21:27:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_35 2026-02-20T21:27:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_39 2026-02-20T21:27:39.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:39.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:39.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:39.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:39.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-20T21:27:39.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:27:39.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:27:39.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:27:39.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:39.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:39.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:40.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-20T21:27:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-20T21:27:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:27:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-20T21:27:40.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-20T21:27:40.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=56990], [object_number=53, tag_tid=1, entry_tid=56989], [object_number=52, tag_tid=1, entry_tid=56988], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:40.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:40.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-20T21:27:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:40.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-20T21:27:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:27:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-20T21:27:40.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-20T21:27:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=56990], [object_number=53, tag_tid=1, entry_tid=56989], [object_number=52, tag_tid=1, entry_tid=56988], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=56990' '!=' 'object_number=54, tag_tid=1, entry_tid=56990' 2026-02-20T21:27:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:40.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:27:40.407 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:40.403+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9110 2026-02-20T21:27:40.408 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:40.404+0000 7f413d8ae640 -1 reset not still connected to 0x55d2821861a0 2026-02-20T21:27:40.408 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:40.404+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:27:40.565 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:40.562+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:27:41.820 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:41.816+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351e8f0 2026-02-20T21:27:41.821 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:41.816+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3555d40 2026-02-20T21:27:41.822 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:41.818+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:27:41.822 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:41.818+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:27:41.822 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:41.818+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:27:42.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_3 2026-02-20T21:27:42.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:42.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:42.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:42.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:42.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_3 ']' 2026-02-20T21:27:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_3' 2026-02-20T21:27:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_3 2026-02-20T21:27:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:42.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_3 2026-02-20T21:27:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:42.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:42.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:42.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:42.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:42.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:42.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:42.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:42.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:42.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 5 closest matches: 2026-02-20T21:27:42.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_30 2026-02-20T21:27:42.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_32 2026-02-20T21:27:42.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_34 2026-02-20T21:27:42.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_36 2026-02-20T21:27:42.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_38 2026-02-20T21:27:42.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_3 2026-02-20T21:27:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:42.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:42.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:42.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:42.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:42.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:42.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:42.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:42.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 3 closest matches: 2026-02-20T21:27:42.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_31 2026-02-20T21:27:42.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_35 2026-02-20T21:27:42.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_39 2026-02-20T21:27:42.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_3 2026-02-20T21:27:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:42.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:42.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:42.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:42.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:42.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:42.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:42.245 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:42.241+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:27:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:42.249 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:42.245+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:27:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_3'] 2026-02-20T21:27:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:27:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_3 2026-02-20T21:27:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:27:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:27:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:27:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:27:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:27:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:27:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:27:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:27:42.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:42.303 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.299+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:27:42.303 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.299+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:27:42.305 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.301+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:27:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:42.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_3 2026-02-20T21:27:42.307 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.302+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:42.307 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.302+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:27:42.307 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.302+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:42.308 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.302+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:42.308 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.302+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:42.308 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.302+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:42.308 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.302+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:42.308 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.302+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:42.308 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.302+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:42.308 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.305+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_3 2026-02-20T21:27:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:42.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '' 2026-02-20T21:27:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:27:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-20T21:27:42.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:42.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:42.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-20T21:27:42.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:42.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:42.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:42.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:42.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:42.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:42.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:42.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:42.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_3 2026-02-20T21:27:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_3 '..*' 2026-02-20T21:27:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:27:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_3.status 2026-02-20T21:27:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_3.status 2026-02-20T21:27:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:42.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_3 2026-02-20T21:27:42.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:27:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:27:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=56990], [object_number=55, tag_tid=1, entry_tid=56987]]], state=connected] 2026-02-20T21:27:42.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.status 2026-02-20T21:27:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:27:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_3 snap 2026-02-20T21:27:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:27:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:27:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:27:42.582 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.578+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f173ad0 2026-02-20T21:27:42.582 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.578+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f3c7790 2026-02-20T21:27:42.583 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.578+0000 7f413d8ae640 -1 reset not still connected to 0x55d2701460d0 2026-02-20T21:27:42.583 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.578+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:42.583 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.579+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107b5f0 2026-02-20T21:27:42.584 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.579+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:42.586 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:42.586 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:42.586 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:42.586 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:42.587 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:27:42.587 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:42.587 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:42.582+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:43.388 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.384+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:43.389 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.384+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:43.389 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.384+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:43.389 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.384+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:43.389 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.385+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:43.390 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.385+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:43.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_3@snap 2026-02-20T21:27:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_3': 2026-02-20T21:27:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:27:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:27:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:27:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1197b2c85f1c 2026-02-20T21:27:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1197b2c85f1c 2026-02-20T21:27:43.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:27:43.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:27:43.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:27:43.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:27:43.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:19:11 2026 2026-02-20T21:27:43.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:19:11 2026 2026-02-20T21:27:43.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:20:11 2026 2026-02-20T21:27:43.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:27:43.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1197b2c85f1c 2026-02-20T21:27:43.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:27:43.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:27:43.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 6ee30674-3f79-4653-88fd-ab8cafcc8a01 2026-02-20T21:27:43.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:27:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_3 snap 2026-02-20T21:27:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:27:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:43.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:27:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.export 2026-02-20T21:27:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_3.export 2026-02-20T21:27:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_3.export 2026-02-20T21:27:43.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_3@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.export 2026-02-20T21:27:43.513 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.509+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:43.570 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.566+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:43.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:27:43.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_3@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_3.export 2026-02-20T21:27:43.632 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:43.628+0000 7f413d8ae640 -1 reset not still connected to 0x55d26f1728f0 2026-02-20T21:27:43.632 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:43.629+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:27:43.635 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:43.631+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:27:43.635 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:43.631+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074b520 2026-02-20T21:27:43.635 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:43.631+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107a1a0 2026-02-20T21:27:43.635 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:43.631+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:27:43.635 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:43.631+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fb2b0 2026-02-20T21:27:43.635 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:43.631+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3ee0 2026-02-20T21:27:43.635 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:43.631+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:43.665 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.661+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:27:43.665 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.661+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:27:43.665 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.661+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:27:43.666 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.663+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:27:43.667 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.663+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:27:43.668 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:43.665+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:27:43.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:27:43.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_3.export 2026-02-20T21:27:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_3.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_3.export 2026-02-20T21:27:43.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:43.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:27:43.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_4 2026-02-20T21:27:43.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_4 snap 2026-02-20T21:27:43.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:43.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:43.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:27:43.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:43.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_4@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:27:44.275 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:44.271+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:27:44.275 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:44.271+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:27:44.276 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:44.272+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:27:44.276 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:44.272+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:44.276 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:44.272+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:45.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:45.242+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:27:45.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:45.242+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:45.246 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:45.242+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:45.559 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:45.554+0000 7f413d8ae640 -1 reset not still connected to 0x55d27074a1a0 2026-02-20T21:27:45.560 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:45.556+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:45.560 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:45.556+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:45.561 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:45.556+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:45.561 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:45.556+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:45.561 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:45.556+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:45.561 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:45.556+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:45.561 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:45.557+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_4 2026-02-20T21:27:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:46.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:27:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_4 started 2026-02-20T21:27:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:27:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:27:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:27:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:46.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:27:47.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_4 started 2026-02-20T21:27:47.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:47.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:47.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:27:47.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:27:47.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:27:47.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:27:47.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_4 2026-02-20T21:27:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:27:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:27:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:27:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:27:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:47.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:27:47.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:47.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:27:47.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:47.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_4 2026-02-20T21:27:47.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:47.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:47.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:47.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:47.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:27:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:47.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:27:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:27:47.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:47.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:47.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:47.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:47.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_4 2026-02-20T21:27:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_42 2026-02-20T21:27:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_46 2026-02-20T21:27:47.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_4 2026-02-20T21:27:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:27:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:27:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:27:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:27:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:27:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:47.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:47.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:47.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_4 2026-02-20T21:27:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_43 2026-02-20T21:27:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_44 2026-02-20T21:27:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:48.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:48.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_4 2026-02-20T21:27:48.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:27:48.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:48.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:48.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:48.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:27:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:27:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:48.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:48.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:27:48.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:27:48.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:48.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:48.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:48.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:48.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_4 2026-02-20T21:27:48.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:27:48.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:27:48.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:27:48.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:27:48.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:27:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:27:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:27:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_4 2026-02-20T21:27:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:27:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:27:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:27:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:27:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:27:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_4 2026-02-20T21:27:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:27:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:27:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:27:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:27:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:27:48.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:27:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:27:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:48.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:27:48.156 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.151+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e340 2026-02-20T21:27:48.156 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.151+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440000 2026-02-20T21:27:48.156 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.151+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef380 2026-02-20T21:27:48.157 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.151+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:48.158 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.154+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:27:48.158 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.154+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:27:48.158 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.154+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:48.159 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.154+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:27:48.159 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.154+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:48.159 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.154+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:48.159 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.154+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:48.159 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.154+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:48.251 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.246+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:48.251 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:48.247+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:48.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_4 2026-02-20T21:27:48.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:48.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:48.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:27:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_4 ']' 2026-02-20T21:27:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_4' 2026-02-20T21:27:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2026-02-20T21:27:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:48.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:48.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2026-02-20T21:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:48.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:48.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:48.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:48.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:48.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2026-02-20T21:27:48.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_42 2026-02-20T21:27:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_46 2026-02-20T21:27:48.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:48.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:48.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_4 2026-02-20T21:27:48.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:48.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:48.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:48.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:48.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:48.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_4 2026-02-20T21:27:48.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:48.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_43 2026-02-20T21:27:48.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_44 2026-02-20T21:27:48.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:48.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_4 2026-02-20T21:27:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:48.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:48.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:48.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:48.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_4 2026-02-20T21:27:48.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:48.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_4 2026-02-20T21:27:48.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:48.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:48.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:48.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '' 2026-02-20T21:27:48.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:48.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:48.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:48.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:48.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-20T21:27:48.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-20T21:27:48.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:48.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:48.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-20T21:27:48.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-20T21:27:48.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-20T21:27:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=2, entry_tid=1], [object_number=56, tag_tid=2, entry_tid=0], [object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338]]], state=connected] 2026-02-20T21:27:48.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338], [object_number=57, tag_tid=1, entry_tid=60337], [object_number=56, tag_tid=1, entry_tid=60336]]], state=connected] 2026-02-20T21:27:48.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:48.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=2, entry_tid=1' 2026-02-20T21:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2026-02-20T21:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2026-02-20T21:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-20T21:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-20T21:27:48.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:48.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:48.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-20T21:27:48.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-20T21:27:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-20T21:27:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=2, entry_tid=1], [object_number=56, tag_tid=2, entry_tid=0], [object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338]]], state=connected] 2026-02-20T21:27:48.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338], [object_number=57, tag_tid=1, entry_tid=60337], [object_number=56, tag_tid=1, entry_tid=60336]]], state=connected] 2026-02-20T21:27:48.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:48.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=59, tag_tid=1, entry_tid=60339' 2026-02-20T21:27:48.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=2, entry_tid=1' -a 'object_number=57, tag_tid=2, entry_tid=1' = 'object_number=59, tag_tid=1, entry_tid=60339' 2026-02-20T21:27:48.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=59, tag_tid=1, entry_tid=60339' '!=' '' 2026-02-20T21:27:48.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:27:48.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=59, tag_tid=1, entry_tid=60339' = '' 2026-02-20T21:27:48.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=59, tag_tid=1, entry_tid=60339' 2026-02-20T21:27:48.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=2, entry_tid=1' 2026-02-20T21:27:48.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:27:48.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:48.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:27:48.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=59, tag_tid=1, entry_tid=60339' 2026-02-20T21:27:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:27:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:27:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=57, tag_tid=2, entry_tid=1' 2026-02-20T21:27:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:27:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:27:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=59, tag_tid=1, entry_tid=60339' 2026-02-20T21:27:48.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:27:48.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:27:48.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=60339 2026-02-20T21:27:48.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 60339 2026-02-20T21:27:48.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:27:48.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:48.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:27:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_4 2026-02-20T21:27:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:27:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_4 ']' 2026-02-20T21:27:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_4' 2026-02-20T21:27:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2026-02-20T21:27:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:48.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:48.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2026-02-20T21:27:48.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:48.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:48.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:48.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:48.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:48.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:48.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:48.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:48.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2026-02-20T21:27:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_42 2026-02-20T21:27:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_46 2026-02-20T21:27:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_4 2026-02-20T21:27:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:48.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_4 2026-02-20T21:27:48.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:48.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_43 2026-02-20T21:27:48.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_44 2026-02-20T21:27:48.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:48.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:48.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_4 2026-02-20T21:27:48.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:48.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:48.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:48.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:48.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:48.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:48.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:48.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:48.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:48.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:48.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:48.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:48.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:48.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:48.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:48.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:48.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:48.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:48.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:48.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:48.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:48.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:48.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_4 2026-02-20T21:27:48.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:48.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_4 2026-02-20T21:27:48.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '' 2026-02-20T21:27:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-20T21:27:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-20T21:27:48.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-20T21:27:48.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-20T21:27:48.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-20T21:27:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=2, entry_tid=1], [object_number=56, tag_tid=2, entry_tid=0], [object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338]]], state=connected] 2026-02-20T21:27:48.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338], [object_number=57, tag_tid=1, entry_tid=60337], [object_number=56, tag_tid=1, entry_tid=60336]]], state=connected] 2026-02-20T21:27:48.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:48.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=2, entry_tid=1' 2026-02-20T21:27:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2026-02-20T21:27:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2026-02-20T21:27:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-20T21:27:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-20T21:27:48.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-20T21:27:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-20T21:27:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-20T21:27:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=2, entry_tid=1], [object_number=56, tag_tid=2, entry_tid=0], [object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338]]], state=connected] 2026-02-20T21:27:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338], [object_number=57, tag_tid=1, entry_tid=60337], [object_number=56, tag_tid=1, entry_tid=60336]]], state=connected] 2026-02-20T21:27:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:48.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=59, tag_tid=1, entry_tid=60339' 2026-02-20T21:27:48.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=2, entry_tid=1' -a 'object_number=57, tag_tid=2, entry_tid=1' = 'object_number=59, tag_tid=1, entry_tid=60339' 2026-02-20T21:27:48.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=59, tag_tid=1, entry_tid=60339' '!=' 'object_number=59, tag_tid=1, entry_tid=60339' 2026-02-20T21:27:48.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:48.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:27:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_4 2026-02-20T21:27:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:49.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:27:49.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:49.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_4 ']' 2026-02-20T21:27:49.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_4' 2026-02-20T21:27:49.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2026-02-20T21:27:49.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:49.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:49.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:49.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:49.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:49.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:49.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:49.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2026-02-20T21:27:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:49.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:49.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:49.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:49.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:49.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:49.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:49.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:49.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:49.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2026-02-20T21:27:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_42 2026-02-20T21:27:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_46 2026-02-20T21:27:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_4 2026-02-20T21:27:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:49.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:49.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_4 2026-02-20T21:27:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_43 2026-02-20T21:27:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_44 2026-02-20T21:27:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_4 2026-02-20T21:27:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:49.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:49.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:49.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:49.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_4 2026-02-20T21:27:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_4 2026-02-20T21:27:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '' 2026-02-20T21:27:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-20T21:27:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-20T21:27:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-20T21:27:49.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-20T21:27:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-20T21:27:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=2, entry_tid=1], [object_number=56, tag_tid=2, entry_tid=0], [object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338]]], state=connected] 2026-02-20T21:27:49.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338], [object_number=57, tag_tid=1, entry_tid=60337], [object_number=56, tag_tid=1, entry_tid=60336]]], state=connected] 2026-02-20T21:27:49.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:49.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=2, entry_tid=1' 2026-02-20T21:27:49.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2026-02-20T21:27:49.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:49.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:49.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:49.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2026-02-20T21:27:49.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:49.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:49.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:49.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:49.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-20T21:27:49.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-20T21:27:49.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:49.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:49.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-20T21:27:49.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:49.607 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:49.603+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:27:49.607 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:49.604+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:27:49.608 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:49.605+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:27:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-20T21:27:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-20T21:27:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=2, entry_tid=1], [object_number=56, tag_tid=2, entry_tid=0], [object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338]]], state=connected] 2026-02-20T21:27:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338], [object_number=57, tag_tid=1, entry_tid=60337], [object_number=56, tag_tid=1, entry_tid=60336]]], state=connected] 2026-02-20T21:27:49.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=59, tag_tid=1, entry_tid=60339' 2026-02-20T21:27:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=2, entry_tid=1' -a 'object_number=57, tag_tid=2, entry_tid=1' = 'object_number=59, tag_tid=1, entry_tid=60339' 2026-02-20T21:27:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=59, tag_tid=1, entry_tid=60339' '!=' 'object_number=59, tag_tid=1, entry_tid=60339' 2026-02-20T21:27:49.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:49.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:27:50.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:50.419+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_4 2026-02-20T21:27:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:27:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_4 ']' 2026-02-20T21:27:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_4' 2026-02-20T21:27:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_4 2026-02-20T21:27:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:50.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:50.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:50.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_4 2026-02-20T21:27:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:50.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:50.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:50.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:50.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:50.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:50.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:50.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_4 2026-02-20T21:27:50.463 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:50.459+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c340 2026-02-20T21:27:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_42 2026-02-20T21:27:50.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_46 2026-02-20T21:27:50.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_4 2026-02-20T21:27:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:50.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:50.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:50.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:50.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:50.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:50.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:50.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_4 2026-02-20T21:27:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 2 closest matches: 2026-02-20T21:27:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_43 2026-02-20T21:27:50.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_44 2026-02-20T21:27:50.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:50.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:50.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_4 2026-02-20T21:27:50.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:50.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:50.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:50.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:50.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:50.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:50.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:50.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:50.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:50.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:50.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:50.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:50.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:50.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:50.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:50.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:50.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:50.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:50.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_4 2026-02-20T21:27:50.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_4 2026-02-20T21:27:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '' 2026-02-20T21:27:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:50.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:50.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:50.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:27:50.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-20T21:27:50.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-20T21:27:50.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-20T21:27:50.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:50.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-20T21:27:50.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-20T21:27:50.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:50.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=2, entry_tid=1], [object_number=56, tag_tid=2, entry_tid=0], [object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338]]], state=connected] 2026-02-20T21:27:50.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=57, tag_tid=2, entry_tid=1], [object_number=56, tag_tid=2, entry_tid=0], [object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338]]], state=connected] 2026-02-20T21:27:50.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:50.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=57, tag_tid=2, entry_tid=1' 2026-02-20T21:27:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_4 2026-02-20T21:27:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_4 '..*' 2026-02-20T21:27:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:27:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:27:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:27:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:27:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_4.status 2026-02-20T21:27:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_4.status 2026-02-20T21:27:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:27:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_4 2026-02-20T21:27:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:50.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 14 2026-02-20T21:27:50.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 14 2026-02-20T21:27:50.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:27:50.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=57, tag_tid=2, entry_tid=1], [object_number=56, tag_tid=2, entry_tid=0], [object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338]]], state=connected] 2026-02-20T21:27:50.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=57, tag_tid=2, entry_tid=1], [object_number=56, tag_tid=2, entry_tid=0], [object_number=59, tag_tid=1, entry_tid=60339], [object_number=58, tag_tid=1, entry_tid=60338]]], state=connected] 2026-02-20T21:27:50.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.status 2026-02-20T21:27:50.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=57, tag_tid=2, entry_tid=1' 2026-02-20T21:27:50.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=57, tag_tid=2, entry_tid=1' -a 'object_number=57, tag_tid=2, entry_tid=1' = 'object_number=57, tag_tid=2, entry_tid=1' 2026-02-20T21:27:50.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:50.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_4 snap 2026-02-20T21:27:50.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:50.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:50.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:27:50.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:50.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:27:50.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:27:50.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:27:51.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_4@snap 2026-02-20T21:27:51.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_4': 2026-02-20T21:27:51.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:27:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:27:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:27:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11b3a67ef30d 2026-02-20T21:27:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11b3a67ef30d 2026-02-20T21:27:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:27:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:27:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:27:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:27:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:19:25 2026 2026-02-20T21:27:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:19:25 2026 2026-02-20T21:27:51.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:20:26 2026 2026-02-20T21:27:51.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:27:51.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11b3a67ef30d 2026-02-20T21:27:51.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:27:51.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:27:51.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: e10148b3-e7cb-4206-8d50-69ed046faa1e 2026-02-20T21:27:51.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:27:51.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:51.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_4 snap 2026-02-20T21:27:51.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:51.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:27:51.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:51.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:27:51.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.export 2026-02-20T21:27:51.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_4.export 2026-02-20T21:27:51.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_4.export 2026-02-20T21:27:51.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_4@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.export 2026-02-20T21:27:51.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:27:51.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_4@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_4.export 2026-02-20T21:27:51.910 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:51.906+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3555d40 2026-02-20T21:27:51.913 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:51.909+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:27:51.914 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:51.909+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:27:51.915 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:51.911+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:27:51.916 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:51.912+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e14abc70 2026-02-20T21:27:51.916 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:51.912+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:27:51.918 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:51.915+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:27:51.919 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:51.915+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:27:51.919 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:51.915+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:27:51.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:27:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_4.export 2026-02-20T21:27:52.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_4.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_4.export 2026-02-20T21:27:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:27:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_5 2026-02-20T21:27:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_5 snap 2026-02-20T21:27:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:52.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:27:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:27:52.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_5@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:27:52.312 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.307+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e14abc70 2026-02-20T21:27:52.312 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.308+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1bc7380 2026-02-20T21:27:52.312 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.308+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e21464e0 2026-02-20T21:27:52.313 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.308+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24aa340 2026-02-20T21:27:52.313 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.308+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e25884e0 2026-02-20T21:27:52.313 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.308+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2f90750 2026-02-20T21:27:52.313 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.308+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ef4651e0 2026-02-20T21:27:52.313 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.308+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f91d1ad0 2026-02-20T21:27:52.637 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.633+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1c06d00 2026-02-20T21:27:52.639 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.635+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2588c30 2026-02-20T21:27:52.640 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.635+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351e8f0 2026-02-20T21:27:52.640 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.635+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3555d40 2026-02-20T21:27:52.640 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.635+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:27:52.640 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.636+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:27:52.641 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.636+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:27:52.703 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.699+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351e8f0 2026-02-20T21:27:52.703 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.699+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3555d40 2026-02-20T21:27:52.704 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.699+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:27:52.704 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.700+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0410 2026-02-20T21:27:52.705 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.702+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:27:52.706 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.702+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:27:52.993 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.988+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3555d40 2026-02-20T21:27:52.994 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.990+0000 7fc9d03ed640 -1 reset not still connected to 0x557711f4cea0 2026-02-20T21:27:52.995 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:52.990+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:27:53.191 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:53.187+0000 7fd67aee6640 -1 reset not still connected to 0x55daae9bb790 2026-02-20T21:27:53.193 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:53.187+0000 7fd67aee6640 -1 reset not still connected to 0x55dab081c0d0 2026-02-20T21:27:53.194 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:53.187+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdea90 2026-02-20T21:27:53.194 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:53.189+0000 7fd67aee6640 -1 reset not still connected to 0x55dab855d110 2026-02-20T21:27:53.194 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:53.189+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:27:53.194 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:53.189+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d040 2026-02-20T21:27:53.202 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:53.198+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:27:53.202 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:53.198+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:27:53.202 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:53.198+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:27:53.202 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:53.198+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b70410 2026-02-20T21:27:53.203 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:53.198+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:27:54.736 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:54.732+0000 7fd67aee6640 -1 reset not still connected to 0x55dab8a61ba0 2026-02-20T21:27:54.739 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:54.735+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b1c70 2026-02-20T21:27:54.739 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:54.735+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c0750 2026-02-20T21:27:54.740 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:54.735+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1ad0 2026-02-20T21:27:54.740 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:54.735+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b68f0 2026-02-20T21:27:54.740 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:54.735+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:27:54.740 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:54.735+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:27:54.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_5 2026-02-20T21:27:54.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:54.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:54.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:27:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_5 started 2026-02-20T21:27:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:27:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:27:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:27:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:54.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:27:55.415 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:55.411+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c680 2026-02-20T21:27:55.416 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:55.412+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1ad0 2026-02-20T21:27:55.433 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:55.429+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e24abad0 2026-02-20T21:27:55.434 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:55.430+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3143450 2026-02-20T21:27:55.434 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:55.430+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:27:55.435 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:55.431+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3fb3860 2026-02-20T21:27:55.435 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:55.431+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fdae7930 2026-02-20T21:27:55.435 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:55.431+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:27:55.435 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:55.431+0000 7fc9d03ed640 -1 reset not still connected to 0x557712f5fad0 2026-02-20T21:27:55.435 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:55.431+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:27:55.480 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.476+0000 7fd67aee6640 -1 reset not still connected to 0x55daafea6000 2026-02-20T21:27:55.482 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.478+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2d2e8f0 2026-02-20T21:27:55.483 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.478+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3441450 2026-02-20T21:27:55.483 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.480+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7d1bc70 2026-02-20T21:27:55.484 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.480+0000 7fd67aee6640 -1 reset not still connected to 0x55dace520d00 2026-02-20T21:27:55.484 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.480+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:27:55.484 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.480+0000 7fd67aee6640 -1 reset not still connected to 0x55dae421fd40 2026-02-20T21:27:55.484 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.480+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27a00 2026-02-20T21:27:55.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_5 started 2026-02-20T21:27:55.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:55.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:55.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:27:55.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:27:55.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:27:55.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:27:55.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_5 2026-02-20T21:27:55.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:27:55.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:27:55.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:55.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:55.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:27:55.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:27:55.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:55.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:27:55.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:27:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_5 2026-02-20T21:27:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:27:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:55.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:27:55.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:55.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:55.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:55.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:27:55.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:55.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:55.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:55.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:55.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:55.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:55.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:55.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:27:55.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:27:55.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_5 2026-02-20T21:27:55.910 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.906+0000 7fd67aee6640 -1 reset not still connected to 0x55dab7fe6680 2026-02-20T21:27:55.912 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.908+0000 7fd67aee6640 -1 reset not still connected to 0x55dac511ca90 2026-02-20T21:27:55.913 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.909+0000 7fd67aee6640 -1 reset not still connected to 0x55daca2b08f0 2026-02-20T21:27:55.913 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.909+0000 7fd67aee6640 -1 reset not still connected to 0x55dace525790 2026-02-20T21:27:55.913 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.909+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa410 2026-02-20T21:27:55.913 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.909+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05aa8f0 2026-02-20T21:27:55.914 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.909+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27da0d0 2026-02-20T21:27:55.914 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.909+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a750 2026-02-20T21:27:55.914 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:27:55.909+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dba0 2026-02-20T21:27:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-20T21:27:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status mirror/image_50 2026-02-20T21:27:55.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_5 2026-02-20T21:27:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:27:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:55.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:55.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:27:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:27:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:55.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:27:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:27:55.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:55.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:55.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:55.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:55.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_5 2026-02-20T21:27:56.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-20T21:27:56.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:56.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_5 2026-02-20T21:27:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:27:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:56.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:27:56.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:27:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:56.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:27:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:27:56.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_5 2026-02-20T21:27:56.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-20T21:27:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:27:56.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_5 2026-02-20T21:27:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:27:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:27:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:27:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:27:56.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:27:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:27:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:27:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:27:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:27:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:27:56.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:27:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:27:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:27:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:27:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:27:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:27:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:27:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:27:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:27:56.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:56.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:56.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:56.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_5 2026-02-20T21:27:56.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:27:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:27:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:27:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:27:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:27:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:27:56.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:27:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:27:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_5 2026-02-20T21:27:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:27:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:27:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:27:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:27:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:27:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_5 2026-02-20T21:27:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:27:56.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:27:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:27:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:27:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:27:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:27:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:27:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:27:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:27:56.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:27:56.158 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:56.154+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d48f0 2026-02-20T21:27:56.158 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:56.154+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbad0 2026-02-20T21:27:56.159 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:56.155+0000 7f413d8ae640 -1 reset not still connected to 0x55d275323040 2026-02-20T21:27:56.159 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:56.155+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750000 2026-02-20T21:27:56.159 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:56.155+0000 7f413d8ae640 -1 reset not still connected to 0x55d275750270 2026-02-20T21:27:56.159 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:56.155+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0dee0 2026-02-20T21:27:56.160 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:56.156+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92a0d0 2026-02-20T21:27:56.245 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.305753204082465) > weight (0.25) 2026-02-20T21:27:56.245 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7996919166651342) > weight (0.25) 2026-02-20T21:27:56.245 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.502128852184511) > weight (0.25) 2026-02-20T21:27:56.245 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-20T21:27:56.246 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-20T21:27:56.246 INFO:tasks.rbd_mirror_thrash:waiting for 2.016140813213961 secs before reviving daemons 2026-02-20T21:27:56.246 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:27:56.244+0000 7f169f8b4640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 53118) UID: 1000 2026-02-20T21:27:56.253 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:27:56.250+0000 7f16980a5640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x555d9145b500 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-20T21:27:56.253 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:27:56.250+0000 7f16980a5640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x555d9145b500 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-20T21:27:56.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-20T21:27:56.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:56.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:27:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:27:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:27:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-20T21:27:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-20T21:27:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-20T21:27:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:27:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:27:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:27:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:27:56.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:27:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:56.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:56.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:56.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-20T21:27:56.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:56.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:56.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-20T21:27:56.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-20T21:27:56.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-20T21:27:56.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:56.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:56.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-20T21:27:56.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:56.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:56.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:56.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:56.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:56.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:56.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:56.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:56.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:56.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:56.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:56.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:56.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:56.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:56.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:56.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:56.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:56.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:56.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:56.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:56.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:56.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:56.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-20T21:27:56.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-20T21:27:56.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:27:56.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:56.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:56.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-02-20T21:27:56.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:56.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:56.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:56.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-02-20T21:27:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-20T21:27:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:27:56.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2026-02-20T21:27:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:27:56.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:27:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:27:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:56.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:56.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:56.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:56.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:27:56.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:27:56.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2026-02-20T21:27:56.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-20T21:27:56.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:56.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:27:57.312 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:27:57.309+0000 7f534d443640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x56325fe53880 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-20T21:27:57.312 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:27:57.309+0000 7f534d443640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x56325fe53880 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-20T21:27:57.313 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:27:57.311+0000 7fb00939c640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55b536e84380 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-20T21:27:57.313 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:27:57.311+0000 7fb00939c640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x55b536e84380 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-20T21:27:57.314 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:27:57.311+0000 7faa4681e640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x561ec322b500 handle_process_entry_safe: failed to commit journal event: (85) Interrupted system call should be restarted 2026-02-20T21:27:57.314 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:27:57.311+0000 7faa4681e640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x561ec322b500 handle_replay_complete: replay encountered an error: (85) Interrupted system call should be restarted 2026-02-20T21:27:57.391 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:27:57.389+0000 7f03c8435640 -1 reset not still connected to 0x55afbc1c64e0 2026-02-20T21:27:57.392 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:27:57.390+0000 7f03c8435640 -1 reset not still connected to 0x55afbc1c7ee0 2026-02-20T21:27:57.393 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:27:57.390+0000 7f03c8435640 -1 reset not still connected to 0x55afd468e0d0 2026-02-20T21:27:57.393 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:27:57.390+0000 7f03c8435640 -1 reset not still connected to 0x55afd799ac30 2026-02-20T21:27:57.393 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:27:57.390+0000 7f03c8435640 -1 reset not still connected to 0x55afd799b1e0 2026-02-20T21:27:57.393 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:27:57.390+0000 7f03c8435640 -1 reset not still connected to 0x55afde1b5450 2026-02-20T21:27:57.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:57.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:57.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-20T21:27:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:57.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:57.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:57.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:57.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-20T21:27:57.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-20T21:27:57.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-20T21:27:57.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:57.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-20T21:27:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:57.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:57.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:57.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-20T21:27:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-20T21:27:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:27:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:57.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-02-20T21:27:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:57.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:57.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:57.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:57.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-02-20T21:27:57.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-20T21:27:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:27:57.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:57.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:57.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2026-02-20T21:27:57.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:27:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:27:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:27:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:27:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:27:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:27:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:27:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:27:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:27:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:57.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:27:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:27:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:27:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:27:57.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:27:57.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:27:57.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2026-02-20T21:27:57.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-20T21:27:57.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:27:57.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:27:58.263 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-20T21:27:58.263 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:27:58.263 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-20T21:27:58.263 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-20T21:27:58.264 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-20T21:27:58.266 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-20T21:27:58.266 INFO:tasks.rbd_mirror_thrash:waiting for 87.49679180672675 secs before thrashing 2026-02-20T21:27:59.151 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.147+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:27:59.151 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.147+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3ea41a0 2026-02-20T21:27:59.151 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.147+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3ea5790 2026-02-20T21:27:59.151 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.147+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2410 2026-02-20T21:27:59.152 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.148+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0680 2026-02-20T21:27:59.153 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.148+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:27:59.153 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.148+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1040 2026-02-20T21:27:59.238 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:59.234+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:27:59.238 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:59.234+0000 7f413d8ae640 -1 reset not still connected to 0x55d282187ee0 2026-02-20T21:27:59.239 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:59.235+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:27:59.239 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:59.235+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:27:59.267 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:59.263+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:27:59.267 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:59.263+0000 7f413d8ae640 -1 reset not still connected to 0x55d282187ee0 2026-02-20T21:27:59.267 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:59.263+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:27:59.267 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:27:59.263+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:27:59.374 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.369+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:27:59.375 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.371+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:27:59.375 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.371+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:27:59.375 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.371+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554d00 2026-02-20T21:27:59.375 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.371+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:27:59.375 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.371+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:27:59.376 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.372+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:27:59.376 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.372+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:27:59.377 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.372+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:27:59.658 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.653+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c4e0 2026-02-20T21:27:59.658 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.653+0000 7fc9d03ed640 -1 reset not still connected to 0x557706d77930 2026-02-20T21:27:59.659 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.655+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0ea0 2026-02-20T21:27:59.659 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:27:59.655+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:27:59.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:27:59.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:27:59.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-20T21:27:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:27:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:27:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:27:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:59.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:59.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:27:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:27:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:27:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:27:59.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:27:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:27:59.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-20T21:27:59.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-20T21:27:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-20T21:27:59.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-20T21:27:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:27:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:27:59.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:27:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:27:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:27:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:27:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:27:59.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:27:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:27:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:27:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:27:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:59.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:27:59.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:27:59.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:27:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:27:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-20T21:27:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-20T21:27:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:27:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:27:59.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:27:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_5 2026-02-20T21:27:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:27:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:27:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:27:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:27:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:27:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:27:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:27:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:27:59.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:27:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:27:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:27:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:27:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:27:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:27:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:27:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:27:59.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:27:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:27:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:27:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:27:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:27:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:27:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:27:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:27:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:27:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:27:59.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:27:59.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:27:59.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:00.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-20T21:28:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:00.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:00.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:00.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_5 2026-02-20T21:28:00.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:28:00.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:00.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:28:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:28:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:28:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:28:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:28:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:28:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:28:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:28:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:28:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:28:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:00.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:28:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:28:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:28:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:28:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:28:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:28:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:28:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:28:00.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:00.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_5'] 2026-02-20T21:28:00.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:00.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-20T21:28:00.582 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:00.575+0000 7f413d8ae640 -1 reset not still connected to 0x55d283728000 2026-02-20T21:28:00.582 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:00.576+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:28:00.582 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:00.576+0000 7f413d8ae640 -1 reset not still connected to 0x55d293de6750 2026-02-20T21:28:00.699 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:00.695+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:00.701 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:00.697+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe410 2026-02-20T21:28:00.701 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:00.697+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806992b0 2026-02-20T21:28:00.702 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:00.698+0000 7f413d8ae640 -1 reset not still connected to 0x55d282187ee0 2026-02-20T21:28:00.702 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:00.698+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:00.702 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:00.698+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:00.797 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:00.793+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07acd00 2026-02-20T21:28:00.797 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:00.793+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c4e0 2026-02-20T21:28:00.798 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:00.793+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:28:02.173 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:02.169+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c4e0 2026-02-20T21:28:02.173 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:02.169+0000 7fc9d03ed640 -1 reset not still connected to 0x557706d77930 2026-02-20T21:28:02.176 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:02.172+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0ea0 2026-02-20T21:28:02.176 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:02.172+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:28:02.319 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:02.314+0000 7f413d8ae640 -1 reset not still connected to 0x55d280699860 2026-02-20T21:28:02.319 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:02.314+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b35f0 2026-02-20T21:28:02.320 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:02.316+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92bc70 2026-02-20T21:28:02.335 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.331+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdf520 2026-02-20T21:28:02.336 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.331+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdfd40 2026-02-20T21:28:02.336 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.331+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:28:02.336 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.331+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef110 2026-02-20T21:28:02.336 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.331+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b6680 2026-02-20T21:28:02.336 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.331+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b7e10 2026-02-20T21:28:02.336 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.332+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:28:02.337 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.332+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:28:02.337 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.332+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b860 2026-02-20T21:28:02.337 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.333+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92bee0 2026-02-20T21:28:02.337 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.333+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d2b0 2026-02-20T21:28:02.337 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.333+0000 7fd67aee6640 -1 reset not still connected to 0x55daead265b0 2026-02-20T21:28:02.338 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.333+0000 7fd67aee6640 -1 reset not still connected to 0x55daead272b0 2026-02-20T21:28:02.338 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:02.333+0000 7fd67aee6640 -1 reset not still connected to 0x55daeca9e410 2026-02-20T21:28:03.281 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:03.277+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:28:03.809 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:03.804+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:03.809 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:03.804+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:03.810 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:03.806+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:03.810 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:03.806+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:03.810 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:03.806+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:03.811 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:03.806+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:03.811 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:03.806+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:03.811 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:03.806+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:03.811 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:03.806+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:04.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-20T21:28:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:04.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:04.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:04.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:04.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:04.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:04.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:04.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:04.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-20T21:28:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-20T21:28:04.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:04.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:04.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-20T21:28:04.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:04.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:04.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:04.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:04.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:04.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:04.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:04.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:04.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:04.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:04.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-20T21:28:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-20T21:28:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:04.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:04.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:04.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:04.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:04.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:04.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=54834], [object_number=53, tag_tid=1, entry_tid=54833], [object_number=52, tag_tid=1, entry_tid=54832], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:04.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:04.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:28:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-20T21:28:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-20T21:28:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:04.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:04.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:04.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:04.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:04.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=54834], [object_number=53, tag_tid=1, entry_tid=54833], [object_number=52, tag_tid=1, entry_tid=54832], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:04.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=54834' '!=' '' 2026-02-20T21:28:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:28:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=54834' = '' 2026-02-20T21:28:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:28:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:28:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:28:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:28:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:28:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:28:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:28:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:04.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:28:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:28:04.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=54834 2026-02-20T21:28:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 54834 2026-02-20T21:28:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:28:04.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:04.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:28:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-20T21:28:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:28:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-20T21:28:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-20T21:28:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-20T21:28:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-20T21:28:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:04.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:04.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:04.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:04.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:04.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:04.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:04.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-20T21:28:04.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-20T21:28:04.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-20T21:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:04.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:04.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:04.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:04.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:04.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:04.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:04.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:04.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-20T21:28:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-20T21:28:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:04.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:04.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=54834], [object_number=53, tag_tid=1, entry_tid=54833], [object_number=52, tag_tid=1, entry_tid=54832], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:04.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:04.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:28:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-20T21:28:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-20T21:28:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:04.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:04.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:04.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:04.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=54834], [object_number=53, tag_tid=1, entry_tid=54833], [object_number=52, tag_tid=1, entry_tid=54832], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:04.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=54834' '!=' 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:28:04.906 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:04.902+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab790 2026-02-20T21:28:04.907 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:04.903+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27db380 2026-02-20T21:28:04.907 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:04.903+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d380 2026-02-20T21:28:04.908 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:04.904+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:28:04.908 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:04.904+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6c4e0 2026-02-20T21:28:04.908 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:04.904+0000 7fd67aee6640 -1 reset not still connected to 0x55daead268f0 2026-02-20T21:28:04.909 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:04.904+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27110 2026-02-20T21:28:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-20T21:28:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:28:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-20T21:28:05.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-20T21:28:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-20T21:28:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:05.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-20T21:28:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:05.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:05.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:05.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-20T21:28:05.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-20T21:28:05.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:05.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:05.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-20T21:28:05.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:05.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:05.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:05.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:05.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-20T21:28:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-20T21:28:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:05.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:05.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:05.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:05.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:05.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:05.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=54834], [object_number=53, tag_tid=1, entry_tid=54833], [object_number=52, tag_tid=1, entry_tid=54832], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:05.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:05.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:28:05.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-20T21:28:05.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:05.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-20T21:28:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:05.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:05.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:05.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:05.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:05.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:05.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=54834], [object_number=53, tag_tid=1, entry_tid=54833], [object_number=52, tag_tid=1, entry_tid=54832], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:05.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:05.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=54834' '!=' 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:05.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:28:05.698 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:05.695+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:05.698 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:05.695+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:05.701 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:05.696+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:05.701 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:05.696+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:05.701 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:05.696+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:05.701 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:05.696+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:05.701 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:05.696+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:06.060 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.056+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c4e0 2026-02-20T21:28:06.061 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.056+0000 7fc9d03ed640 -1 reset not still connected to 0x557706d77930 2026-02-20T21:28:06.061 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.057+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:28:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-20T21:28:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:28:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-20T21:28:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-20T21:28:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-20T21:28:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-20T21:28:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:06.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:06.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:06.080 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:06.076+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:06.120 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.116+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:28:06.120 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.116+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:06.121 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.117+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:06.122 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.117+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:06.122 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.117+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:06.122 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.117+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:06.122 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.117+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:06.122 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.117+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-20T21:28:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-20T21:28:06.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:06.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:06.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-20T21:28:06.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:06.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:06.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:06.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:06.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:06.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:06.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:06.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:06.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-20T21:28:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-20T21:28:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:06.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:06.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:06.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:06.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:06.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:06.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:06.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:06.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=54834], [object_number=53, tag_tid=1, entry_tid=54833], [object_number=52, tag_tid=1, entry_tid=54832], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:06.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:06.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:28:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-20T21:28:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-20T21:28:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:06.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:06.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:06.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:06.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:06.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=54834], [object_number=53, tag_tid=1, entry_tid=54833], [object_number=52, tag_tid=1, entry_tid=54832], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:06.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:06.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=54834' '!=' 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:06.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:28:06.934 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.930+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:28:06.937 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.933+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:06.938 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.934+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:06.938 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.934+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:06.938 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.934+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:06.938 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.934+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:06.938 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.934+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:06.939 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.935+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:06.939 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.935+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:06.939 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:06.935+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:07.404 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.400+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:28:07.405 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.401+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:07.407 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.402+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:07.407 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.402+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:07.407 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.402+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:07.407 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.402+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:07.407 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.403+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:07.408 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.404+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:07.408 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.404+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:07.408 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.404+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:07.491 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.486+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07acd00 2026-02-20T21:28:07.491 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.486+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b705b0 2026-02-20T21:28:07.491 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.486+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c4e0 2026-02-20T21:28:07.491 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.486+0000 7fc9d03ed640 -1 reset not still connected to 0x557706d77930 2026-02-20T21:28:07.491 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.486+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0ea0 2026-02-20T21:28:07.491 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:07.487+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:28:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-20T21:28:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:28:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-20T21:28:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-20T21:28:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-20T21:28:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-20T21:28:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:07.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:07.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-20T21:28:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-20T21:28:07.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-20T21:28:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:07.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:07.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-20T21:28:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:07.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-20T21:28:07.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:07.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:07.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:07.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:07.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:07.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:07.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:07.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:07.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:07.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:08.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=54834], [object_number=53, tag_tid=1, entry_tid=54833], [object_number=52, tag_tid=1, entry_tid=54832], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:08.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:08.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:28:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-20T21:28:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-20T21:28:08.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:08.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:08.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:08.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:08.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=54834], [object_number=53, tag_tid=1, entry_tid=54833], [object_number=52, tag_tid=1, entry_tid=54832], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:08.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:08.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:08.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:08.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=54834' '!=' 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:08.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:08.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:28:09.060 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.056+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab790 2026-02-20T21:28:09.064 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.059+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:28:09.065 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.060+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27db380 2026-02-20T21:28:09.065 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.060+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d380 2026-02-20T21:28:09.065 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.060+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:28:09.065 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.060+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6c4e0 2026-02-20T21:28:09.065 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.060+0000 7fd67aee6640 -1 reset not still connected to 0x55daead268f0 2026-02-20T21:28:09.065 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.060+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27110 2026-02-20T21:28:09.389 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.385+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab790 2026-02-20T21:28:09.394 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.390+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:28:09.394 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.390+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27db380 2026-02-20T21:28:09.394 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.390+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d380 2026-02-20T21:28:09.394 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.390+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:28:09.395 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.390+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6c4e0 2026-02-20T21:28:09.395 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.391+0000 7fd67aee6640 -1 reset not still connected to 0x55daead268f0 2026-02-20T21:28:09.395 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:09.391+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27110 2026-02-20T21:28:09.548 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:09.544+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d4c30 2026-02-20T21:28:09.549 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:09.545+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:28:09.549 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:09.545+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:09.550 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:09.545+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe410 2026-02-20T21:28:09.550 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:09.546+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806992b0 2026-02-20T21:28:09.551 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:09.547+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:09.551 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:09.547+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:09.712 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:09.708+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-20T21:28:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:28:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-20T21:28:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-20T21:28:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-20T21:28:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-20T21:28:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:10.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:10.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:10.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:10.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:10.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-20T21:28:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-20T21:28:10.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-20T21:28:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:10.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:10.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:10.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:10.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:10.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-20T21:28:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:10.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-20T21:28:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:10.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=54834], [object_number=53, tag_tid=1, entry_tid=54833], [object_number=52, tag_tid=1, entry_tid=54832], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:28:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-20T21:28:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-20T21:28:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:10.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:10.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:10.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:10.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=54, tag_tid=1, entry_tid=54834], [object_number=53, tag_tid=1, entry_tid=54833], [object_number=52, tag_tid=1, entry_tid=54832], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:10.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=54, tag_tid=1, entry_tid=54834' '!=' 'object_number=54, tag_tid=1, entry_tid=54834' 2026-02-20T21:28:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:10.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:28:10.499 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:10.494+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:10.499 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:10.494+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:10.499 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:10.494+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:10.499 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:10.494+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:10.499 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:10.494+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:10.499 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:10.494+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:10.500 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:10.495+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:10.500 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:10.496+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:10.622 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:10.618+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab790 2026-02-20T21:28:10.625 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:10.621+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:28:10.625 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:10.621+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d380 2026-02-20T21:28:10.625 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:10.621+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:28:10.625 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:10.621+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6c4e0 2026-02-20T21:28:10.625 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:10.621+0000 7fd67aee6640 -1 reset not still connected to 0x55daead268f0 2026-02-20T21:28:10.626 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:10.621+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27110 2026-02-20T21:28:11.863 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:11.858+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcab60 2026-02-20T21:28:11.863 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:11.858+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e2589380 2026-02-20T21:28:11.863 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:11.858+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351f380 2026-02-20T21:28:11.863 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:11.858+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e351f520 2026-02-20T21:28:11.863 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:11.858+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c1450 2026-02-20T21:28:11.863 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:11.858+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3ea41a0 2026-02-20T21:28:11.864 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:11.858+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:28:11.864 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:11.858+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1040 2026-02-20T21:28:12.068 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:12.064+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423c70 2026-02-20T21:28:12.071 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:12.067+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440d00 2026-02-20T21:28:12.071 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:12.067+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef2b0 2026-02-20T21:28:12.071 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:12.067+0000 7fd67aee6640 -1 reset not still connected to 0x55dab804ec30 2026-02-20T21:28:12.071 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:12.067+0000 7fd67aee6640 -1 reset not still connected to 0x55dab804f450 2026-02-20T21:28:12.071 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:12.067+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c08f0 2026-02-20T21:28:12.072 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:12.067+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:12.177 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.173+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:28:12.178 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.173+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:12.178 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.174+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe410 2026-02-20T21:28:12.180 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.176+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806992b0 2026-02-20T21:28:12.180 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.176+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:12.180 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.176+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:12.189 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.185+0000 7f413d8ae640 -1 reset not still connected to 0x55d270ea1a00 2026-02-20T21:28:12.189 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.185+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe9c0 2026-02-20T21:28:12.189 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.185+0000 7f413d8ae640 -1 reset not still connected to 0x55d280699860 2026-02-20T21:28:12.190 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.186+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b25b0 2026-02-20T21:28:12.190 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.186+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b35f0 2026-02-20T21:28:12.190 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.186+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c5b0 2026-02-20T21:28:12.190 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.186+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92bc70 2026-02-20T21:28:12.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_5 2026-02-20T21:28:12.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:12.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:12.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:28:12.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:12.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_5 ']' 2026-02-20T21:28:12.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_5' 2026-02-20T21:28:12.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_5 2026-02-20T21:28:12.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:12.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:12.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:12.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:12.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:12.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:12.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:12.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_5 2026-02-20T21:28:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:12.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:12.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:12.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:12.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:12.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:12.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:12.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:no valid command found; 1 closest matches: 2026-02-20T21:28:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush mirror/image_50 2026-02-20T21:28:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:12.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_5 2026-02-20T21:28:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:12.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:12.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:12.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:12.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:12.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:12.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:12.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_5 2026-02-20T21:28:12.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:12.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_5 2026-02-20T21:28:12.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:12.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:12.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:12.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '' 2026-02-20T21:28:12.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:12.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:12.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:12.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:12.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:12.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:12.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:12.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:12.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:12.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:12.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:12.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:12.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:28:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_5 2026-02-20T21:28:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:12.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_5 '..*' 2026-02-20T21:28:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:28:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_5.status 2026-02-20T21:28:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_5.status 2026-02-20T21:28:12.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_5 2026-02-20T21:28:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:12.419 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d270ea1a00 2026-02-20T21:28:12.419 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.414+0000 7f413d8ae640 -1 reset not still connected to 0x55d280699860 2026-02-20T21:28:12.421 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.417+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b35f0 2026-02-20T21:28:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 13 2026-02-20T21:28:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 13 2026-02-20T21:28:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:12.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=53, tag_tid=2, entry_tid=1], [object_number=52, tag_tid=2, entry_tid=0], [object_number=54, tag_tid=1, entry_tid=54834], [object_number=55, tag_tid=1, entry_tid=54831]]], state=connected] 2026-02-20T21:28:12.422 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:12.418+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92bc70 2026-02-20T21:28:12.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.status 2026-02-20T21:28:12.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:28:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=53, tag_tid=2, entry_tid=1' -a 'object_number=53, tag_tid=2, entry_tid=1' = 'object_number=53, tag_tid=2, entry_tid=1' 2026-02-20T21:28:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_5 snap 2026-02-20T21:28:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:28:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:28:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:28:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:28:12.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:28:13.343 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:13.339+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423c70 2026-02-20T21:28:13.344 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:13.340+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef2b0 2026-02-20T21:28:13.345 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:13.341+0000 7fd67aee6640 -1 reset not still connected to 0x55dab804ec30 2026-02-20T21:28:13.345 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:13.341+0000 7fd67aee6640 -1 reset not still connected to 0x55dab804f450 2026-02-20T21:28:13.346 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:13.341+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c08f0 2026-02-20T21:28:13.348 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:13.344+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_5@snap 2026-02-20T21:28:13.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_5': 2026-02-20T21:28:13.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:28:13.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:28:13.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:28:13.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ecab99d458 2026-02-20T21:28:13.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ecab99d458 2026-02-20T21:28:13.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:28:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:28:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:28:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:28:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:19:39 2026 2026-02-20T21:28:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:19:39 2026 2026-02-20T21:28:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:20:40 2026 2026-02-20T21:28:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:28:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ecab99d458 2026-02-20T21:28:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:28:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:28:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 42180d38-7f35-4066-9e9e-83ce0cdd58fa 2026-02-20T21:28:13.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:28:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_5 snap 2026-02-20T21:28:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:28:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:28:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:28:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.export 2026-02-20T21:28:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_5.export 2026-02-20T21:28:13.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_5.export 2026-02-20T21:28:13.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_5@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.export 2026-02-20T21:28:13.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:28:13.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_5@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_5.export 2026-02-20T21:28:13.622 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:13.618+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:28:13.623 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:13.619+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:28:13.625 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:13.621+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:28:13.625 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:13.621+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:28:13.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:28:13.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_5.export 2026-02-20T21:28:13.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_5.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_5.export 2026-02-20T21:28:13.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:13.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:28:13.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_6 2026-02-20T21:28:13.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_6 snap 2026-02-20T21:28:13.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:28:13.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:13.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:13.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:28:13.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_6@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:28:14.133 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:14.129+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:14.133 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:14.129+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:14.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1ea90 2026-02-20T21:28:14.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdf520 2026-02-20T21:28:14.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdfd40 2026-02-20T21:28:14.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:28:14.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef110 2026-02-20T21:28:14.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b6680 2026-02-20T21:28:14.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b7e10 2026-02-20T21:28:14.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:28:14.444 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:28:14.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b860 2026-02-20T21:28:14.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d2b0 2026-02-20T21:28:14.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55daead265b0 2026-02-20T21:28:14.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55daead272b0 2026-02-20T21:28:14.445 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:14.439+0000 7fd67aee6640 -1 reset not still connected to 0x55daeca9e410 2026-02-20T21:28:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_6 2026-02-20T21:28:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_6 started 2026-02-20T21:28:14.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:28:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:28:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:14.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:28:15.266 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.262+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:15.266 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.262+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:15.267 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.263+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:15.268 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.264+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:15.269 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.265+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:15.395 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:15.391+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:15.399 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:15.392+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:15.406 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.402+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:15.408 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.404+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:15.409 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.405+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:15.409 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.405+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:15.411 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.407+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:15.411 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.407+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:15.411 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.407+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:15.411 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:15.407+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:15.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_6 started 2026-02-20T21:28:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:28:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:28:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:28:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_6 2026-02-20T21:28:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:28:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:28:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:28:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:28:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:28:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:28:15.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:15.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_6 2026-02-20T21:28:15.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:28:15.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:15.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:15.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:15.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:15.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:15.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:15.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:15.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:28:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:15.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_6 2026-02-20T21:28:15.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:15.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:28:15.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_6 2026-02-20T21:28:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:28:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:28:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:15.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:15.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_6 2026-02-20T21:28:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:28:16.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:16.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:16.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_6 2026-02-20T21:28:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:28:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:16.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:28:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:28:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:28:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:28:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:16.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:16.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:16.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:16.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_6 2026-02-20T21:28:16.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:28:16.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:28:16.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:28:16.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:28:16.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:28:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:28:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:28:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_6 2026-02-20T21:28:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:28:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:28:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:28:16.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:28:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:28:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_6 2026-02-20T21:28:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:28:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:28:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:28:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:28:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:28:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:28:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:28:16.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:16.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:28:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-20T21:28:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-20T21:28:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-20T21:28:16.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-20T21:28:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:16.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-20T21:28:16.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:16.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:16.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:16.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-20T21:28:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:16.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:16.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:16.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:16.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:16.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:16.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:16.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:16.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-02-20T21:28:16.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:16.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:16.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:16.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:16.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:16.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:16.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:16.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:16.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:16.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:16.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:16.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:16.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:16.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:16.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:16.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:16.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:16.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:16.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:16.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:16.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:16.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:16.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:16.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:16.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:16.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:16.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:16.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:16.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:16.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-20T21:28:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-20T21:28:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:16.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:16.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:16.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:16.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=52273], [object_number=48, tag_tid=1, entry_tid=52272], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:16.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:28:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-20T21:28:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:16.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-20T21:28:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:16.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:16.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=52273], [object_number=48, tag_tid=1, entry_tid=52272], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:16.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=52273' '!=' '' 2026-02-20T21:28:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:28:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=52273' = '' 2026-02-20T21:28:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:28:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:28:16.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:16.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:28:16.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:16.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:28:16.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:28:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:28:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:28:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:28:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:28:16.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=52273 2026-02-20T21:28:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 52273 2026-02-20T21:28:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:28:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:28:16.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-20T21:28:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-20T21:28:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-20T21:28:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-20T21:28:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:16.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:16.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:16.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:16.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:16.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:16.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:16.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:16.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-20T21:28:16.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:16.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:16.798 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.794+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:28:16.799 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.795+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:16.801 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.797+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:16.801 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.797+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:16.801 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.797+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:16.801 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.797+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:16.801 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.797+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:16.802 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.797+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:16.802 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.797+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:16.802 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.797+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:16.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:16.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:16.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-20T21:28:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:16.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:16.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:16.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:16.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-02-20T21:28:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:16.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:16.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-20T21:28:16.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:16.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:16.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:16.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-20T21:28:16.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:16.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:16.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:16.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=52273], [object_number=48, tag_tid=1, entry_tid=52272], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:28:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-20T21:28:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-20T21:28:16.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:16.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:16.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:16.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:16.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:16.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:16.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:16.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:16.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:16.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:16.984 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.980+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7790 2026-02-20T21:28:16.985 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.981+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:16.986 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.982+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:16.986 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.982+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:16.986 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.982+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:16.986 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.982+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:16.986 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.982+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:16.986 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.983+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:16.986 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.983+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:16.987 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:16.983+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:16.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=52273], [object_number=48, tag_tid=1, entry_tid=52272], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:16.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:16.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=52273' '!=' 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:17.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:28:17.023 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:17.019+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-20T21:28:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-20T21:28:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-20T21:28:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-20T21:28:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:17.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-20T21:28:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:17.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:17.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:17.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:17.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:17.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:17.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:17.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:17.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:17.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-20T21:28:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:17.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:17.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:17.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:17.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:17.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-02-20T21:28:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:17.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:17.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:17.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:17.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:17.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:17.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:17.539 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:17.535+0000 7fc9d03ed640 -1 reset not still connected to 0x557706d77930 2026-02-20T21:28:17.540 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:17.536+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0ea0 2026-02-20T21:28:17.540 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:17.536+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:28:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-20T21:28:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-20T21:28:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:17.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:17.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:17.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:17.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:17.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:17.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:17.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:17.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:17.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:17.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=52273], [object_number=48, tag_tid=1, entry_tid=52272], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:17.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:28:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-20T21:28:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-20T21:28:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:17.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:17.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:17.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:17.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:17.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:17.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:17.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:17.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=52273], [object_number=48, tag_tid=1, entry_tid=52272], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:17.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=52273' '!=' 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:28:17.819 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:17.815+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:17.822 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.908903511683992) > weight (0.25) 2026-02-20T21:28:17.822 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-20T21:28:17.822 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-20T21:28:17.822 INFO:tasks.rbd_mirror_thrash:waiting for 9.251513233565928 secs before reviving daemons 2026-02-20T21:28:17.823 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:28:17.820+0000 7f1ecde03640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 89174) UID: 1000 2026-02-20T21:28:18.065 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:18.061+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:18.110 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:18.106+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07acd00 2026-02-20T21:28:18.112 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:18.108+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c4e0 2026-02-20T21:28:18.112 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:18.108+0000 7fc9d03ed640 -1 reset not still connected to 0x557706d77930 2026-02-20T21:28:18.112 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:18.108+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0ea0 2026-02-20T21:28:18.115 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:18.111+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:28:18.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-20T21:28:18.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:18.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:18.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:18.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:18.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-20T21:28:18.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-20T21:28:18.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-20T21:28:18.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:18.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:18.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:18.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:18.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:18.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:18.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:18.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:18.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:18.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-20T21:28:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:18.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:18.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:18.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:18.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:18.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:18.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:18.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:18.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:18.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:18.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:18.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:18.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:18.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:18.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:18.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:18.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:18.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-20T21:28:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:18.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:18.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:18.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:18.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-02-20T21:28:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:18.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:18.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:18.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:18.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:18.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:18.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:18.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-20T21:28:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-20T21:28:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:18.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:18.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:18.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:18.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:18.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:18.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=52273], [object_number=48, tag_tid=1, entry_tid=52272], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:18.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:18.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-20T21:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-20T21:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:18.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:18.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:18.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=52273], [object_number=48, tag_tid=1, entry_tid=52272], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=52273' '!=' 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:18.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:28:19.762 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:19.757+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:28:19.762 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:19.757+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:19.762 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:19.757+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe410 2026-02-20T21:28:19.762 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:19.757+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806992b0 2026-02-20T21:28:19.762 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:19.757+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:19.762 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:19.757+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:20.063 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:20.059+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab450 2026-02-20T21:28:20.063 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:20.059+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab790 2026-02-20T21:28:20.064 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:20.060+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27db380 2026-02-20T21:28:20.064 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:20.060+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d380 2026-02-20T21:28:20.064 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:20.060+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:28:20.064 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:20.060+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6c4e0 2026-02-20T21:28:20.064 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:20.060+0000 7fd67aee6640 -1 reset not still connected to 0x55daead268f0 2026-02-20T21:28:20.064 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:20.060+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27110 2026-02-20T21:28:20.185 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:20.182+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:20.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-20T21:28:20.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-20T21:28:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-20T21:28:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-20T21:28:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:20.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:20.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-20T21:28:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:20.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:20.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:20.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:20.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:20.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:20.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:20.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:20.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-20T21:28:20.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:20.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:20.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:20.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:20.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:20.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:20.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:20.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:20.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:20.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:20.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:20.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-02-20T21:28:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:20.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:20.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:20.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:20.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:20.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-20T21:28:20.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:20.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:20.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:20.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-20T21:28:20.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:20.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:20.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:20.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:20.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:20.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:20.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:20.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:20.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:20.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:20.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=52273], [object_number=48, tag_tid=1, entry_tid=52272], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:20.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:20.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:28:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-20T21:28:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:20.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-20T21:28:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:20.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:20.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=52273], [object_number=48, tag_tid=1, entry_tid=52272], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:20.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:20.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:20.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:20.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=52273' '!=' 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:20.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:20.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:28:20.996 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:20.992+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:28:20.997 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:20.992+0000 7f413d8ae640 -1 reset not still connected to 0x55d293de6750 2026-02-20T21:28:21.425 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:21.421+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:28:21.427 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:21.423+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6c4e0 2026-02-20T21:28:21.430 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:21.426+0000 7fd67aee6640 -1 reset not still connected to 0x55daead268f0 2026-02-20T21:28:21.430 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:21.426+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27110 2026-02-20T21:28:22.555 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:22.551+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:28:22.555 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:22.551+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d4c30 2026-02-20T21:28:22.556 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:22.552+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:28:22.556 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:22.552+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:22.556 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:22.552+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe410 2026-02-20T21:28:22.557 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:22.553+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806992b0 2026-02-20T21:28:22.557 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:22.553+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:22.557 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:22.553+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-20T21:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-20T21:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-20T21:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-20T21:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-20T21:28:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:22.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:22.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:22.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:22.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-20T21:28:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:22.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:22.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:22.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:22.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:22.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:22.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:22.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:22.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-02-20T21:28:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:22.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:22.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:22.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:22.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:22.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:22.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:22.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-20T21:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-20T21:28:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:22.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:22.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:22.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=52273], [object_number=48, tag_tid=1, entry_tid=52272], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:22.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:28:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-20T21:28:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-20T21:28:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:22.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:22.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=1, entry_tid=52273], [object_number=48, tag_tid=1, entry_tid=52272], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:22.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=49, tag_tid=1, entry_tid=52273' '!=' 'object_number=49, tag_tid=1, entry_tid=52273' 2026-02-20T21:28:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:28:23.020 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.016+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1c06d00 2026-02-20T21:28:23.021 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.017+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:28:23.022 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.019+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07acd00 2026-02-20T21:28:23.023 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.019+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b705b0 2026-02-20T21:28:23.024 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.020+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b712b0 2026-02-20T21:28:23.025 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.020+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c4e0 2026-02-20T21:28:23.025 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.020+0000 7fc9d03ed640 -1 reset not still connected to 0x557706d77930 2026-02-20T21:28:23.025 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.020+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0ea0 2026-02-20T21:28:23.025 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.020+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:28:23.135 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.131+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1c06d00 2026-02-20T21:28:23.135 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.131+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e39c15f0 2026-02-20T21:28:23.136 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.133+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07acd00 2026-02-20T21:28:23.136 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.133+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b705b0 2026-02-20T21:28:23.138 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.134+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b712b0 2026-02-20T21:28:23.138 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.134+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c4e0 2026-02-20T21:28:23.140 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.136+0000 7fc9d03ed640 -1 reset not still connected to 0x557706d77930 2026-02-20T21:28:23.140 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.136+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0ea0 2026-02-20T21:28:23.140 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:23.136+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:28:23.967 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:23.963+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:23.970 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:23.965+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:24.154 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:24.150+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:24.154 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:24.150+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:24.161 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:24.157+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe410 2026-02-20T21:28:24.161 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:24.157+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806992b0 2026-02-20T21:28:24.161 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:24.157+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:24.162 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:24.157+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:24.422 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:24.418+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:28:24.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:24.418+0000 7fd67aee6640 -1 reset not still connected to 0x55daead268f0 2026-02-20T21:28:24.423 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:24.418+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27110 2026-02-20T21:28:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_6 2026-02-20T21:28:24.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_6 ']' 2026-02-20T21:28:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_6' 2026-02-20T21:28:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_6 2026-02-20T21:28:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:24.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:24.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_6 2026-02-20T21:28:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:24.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:24.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:24.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:24.827 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:24.823+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107bad0 2026-02-20T21:28:24.829 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:24.825+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9d40 2026-02-20T21:28:24.829 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:24.825+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbd40 2026-02-20T21:28:24.829 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:24.825+0000 7f413d8ae640 -1 reset not still connected to 0x55d280699520 2026-02-20T21:28:24.829 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:24.825+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:28:24.829 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:24.825+0000 7f413d8ae640 -1 reset not still connected to 0x55d293de6750 2026-02-20T21:28:24.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:24.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:24.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_6 2026-02-20T21:28:24.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:24.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:24.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:24.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:24.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:24.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:24.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_6'] 2026-02-20T21:28:24.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:24.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_6 2026-02-20T21:28:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:24.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_6 2026-02-20T21:28:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_6 2026-02-20T21:28:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '' 2026-02-20T21:28:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:25.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:25.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:25.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:25.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:25.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:25.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:25.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:28:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_6 2026-02-20T21:28:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_6 '..*' 2026-02-20T21:28:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:28:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_6.status 2026-02-20T21:28:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_6.status 2026-02-20T21:28:25.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:25.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:25.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_6 2026-02-20T21:28:25.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:28:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:28:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:25.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=51, tag_tid=1, entry_tid=52271], [object_number=50, tag_tid=1, entry_tid=52270]]], state=connected] 2026-02-20T21:28:25.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.status 2026-02-20T21:28:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:28:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:28:25.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_6 snap 2026-02-20T21:28:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:28:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:28:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:28:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:28:25.343 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:25.339+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9d40 2026-02-20T21:28:25.343 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:25.339+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbd40 2026-02-20T21:28:25.346 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:25.342+0000 7f413d8ae640 -1 reset not still connected to 0x55d280699520 2026-02-20T21:28:25.346 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:25.342+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:28:25.346 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:25.342+0000 7f413d8ae640 -1 reset not still connected to 0x55d293de6750 2026-02-20T21:28:25.422 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:25.418+0000 7f413d8ae640 -1 reset not still connected to 0x55d27107bad0 2026-02-20T21:28:25.423 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:25.419+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9d40 2026-02-20T21:28:25.423 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:25.419+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5ad0 2026-02-20T21:28:25.424 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:25.420+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbd40 2026-02-20T21:28:25.425 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:25.421+0000 7f413d8ae640 -1 reset not still connected to 0x55d280699520 2026-02-20T21:28:25.425 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:25.421+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:28:25.425 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:25.421+0000 7f413d8ae640 -1 reset not still connected to 0x55d293de6750 2026-02-20T21:28:25.974 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:25.970+0000 7fd67aee6640 -1 reset not still connected to 0x55dab2fdfd40 2026-02-20T21:28:25.974 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:25.970+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:28:25.974 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:25.970+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef110 2026-02-20T21:28:25.975 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:25.970+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b6680 2026-02-20T21:28:25.976 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:25.972+0000 7fd67aee6640 -1 reset not still connected to 0x55dad36b7e10 2026-02-20T21:28:25.976 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:25.972+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92a820 2026-02-20T21:28:25.976 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:25.972+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:28:25.976 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:25.972+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b860 2026-02-20T21:28:25.976 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:25.972+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6d2b0 2026-02-20T21:28:25.977 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:25.973+0000 7fd67aee6640 -1 reset not still connected to 0x55daead265b0 2026-02-20T21:28:25.977 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:25.973+0000 7fd67aee6640 -1 reset not still connected to 0x55daead272b0 2026-02-20T21:28:25.977 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:25.973+0000 7fd67aee6640 -1 reset not still connected to 0x55daeca9e410 2026-02-20T21:28:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_6@snap 2026-02-20T21:28:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_6': 2026-02-20T21:28:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:28:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:28:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:28:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ffd650329c 2026-02-20T21:28:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ffd650329c 2026-02-20T21:28:26.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:28:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:28:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:28:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:28:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:19:52 2026 2026-02-20T21:28:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:19:52 2026 2026-02-20T21:28:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:20:53 2026 2026-02-20T21:28:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:28:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ffd650329c 2026-02-20T21:28:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:28:26.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:28:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: a25a2c7e-574a-4526-a547-9af26d6516bb 2026-02-20T21:28:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:28:26.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:26.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_6 snap 2026-02-20T21:28:26.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:26.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:28:26.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:28:26.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:28:26.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.export 2026-02-20T21:28:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_6.export 2026-02-20T21:28:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_6.export 2026-02-20T21:28:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_6@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.export 2026-02-20T21:28:26.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:28:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_6@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_6.export 2026-02-20T21:28:26.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:28:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_6.export 2026-02-20T21:28:26.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_6.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_6.export 2026-02-20T21:28:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:28:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_7 2026-02-20T21:28:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_7 snap 2026-02-20T21:28:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:28:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:28:26.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_7@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:28:26.744 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:26.739+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:28:26.744 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:26.740+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:26.745 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:26.741+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe410 2026-02-20T21:28:26.745 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:26.741+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806992b0 2026-02-20T21:28:26.745 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:26.741+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:26.747 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:26.743+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:26.881 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:26.877+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:26.882 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:26.878+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:26.882 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:26.878+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:27.051 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:27.047+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:27.052 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:27.048+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:27.052 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:27.048+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:27.074 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-20T21:28:27.075 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:28:27.075 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-20T21:28:27.075 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-20T21:28:27.075 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-20T21:28:27.078 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-20T21:28:27.078 INFO:tasks.rbd_mirror_thrash:waiting for 74.2317235390725 secs before thrashing 2026-02-20T21:28:27.404 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:27.400+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:28.398 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:28.394+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07acd00 2026-02-20T21:28:28.400 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:28.396+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b705b0 2026-02-20T21:28:28.400 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:28.396+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c4e0 2026-02-20T21:28:28.400 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:28.396+0000 7fc9d03ed640 -1 reset not still connected to 0x557706d77930 2026-02-20T21:28:28.400 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:28.396+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0ea0 2026-02-20T21:28:28.400 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:28.396+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:28:28.917 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:28.913+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_7 2026-02-20T21:28:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:29.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_7 started 2026-02-20T21:28:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:28:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:28:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:29.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:28:29.261 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:29.257+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:28:29.261 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:29.257+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:29.261 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:29.257+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe410 2026-02-20T21:28:29.261 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:29.257+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806992b0 2026-02-20T21:28:29.261 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:29.257+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:29.261 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:29.257+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:29.808 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:29.804+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:28:29.809 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:29.804+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:29.810 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:29.806+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe410 2026-02-20T21:28:29.811 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:29.807+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806992b0 2026-02-20T21:28:29.811 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:29.807+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:29.811 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:29.807+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:29.912 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:29.907+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:28:29.912 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:29.907+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6c4e0 2026-02-20T21:28:29.912 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:29.907+0000 7fd67aee6640 -1 reset not still connected to 0x55daead268f0 2026-02-20T21:28:29.912 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:29.908+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27110 2026-02-20T21:28:30.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_7 started 2026-02-20T21:28:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:28:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:28:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:28:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_7 2026-02-20T21:28:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:28:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:28:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:28:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:28:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:28:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:28:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_7 2026-02-20T21:28:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:28:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:28:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:28:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:28:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:28:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:28:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:28:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:28:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:28:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:30.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_7 2026-02-20T21:28:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-20T21:28:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:28:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_7 2026-02-20T21:28:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:28:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:30.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:28:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:28:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:28:30.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:28:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:28:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:28:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:28:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:28:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:28:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:28:30.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:30.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:30.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:30.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:30.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_7 2026-02-20T21:28:30.121 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:30.117+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e820 2026-02-20T21:28:30.122 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:30.118+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423c70 2026-02-20T21:28:30.122 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:30.118+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3055110 2026-02-20T21:28:30.123 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:30.119+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef2b0 2026-02-20T21:28:30.123 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:30.119+0000 7fd67aee6640 -1 reset not still connected to 0x55dab804ec30 2026-02-20T21:28:30.123 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:30.119+0000 7fd67aee6640 -1 reset not still connected to 0x55dab804f450 2026-02-20T21:28:30.124 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:30.119+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c08f0 2026-02-20T21:28:30.124 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:30.120+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:30.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:28:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:28:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:28:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:28:30.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:28:30.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:28:30.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:28:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_7 2026-02-20T21:28:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:28:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:28:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:28:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:28:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:28:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_7 2026-02-20T21:28:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:28:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:28:30.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:28:30.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:28:30.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:30.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:28:30.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:28:30.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:28:30.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:30.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:28:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-20T21:28:30.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:30.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:30.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:30.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:30.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-20T21:28:30.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-20T21:28:30.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-20T21:28:30.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:30.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:30.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:30.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-20T21:28:30.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:30.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:30.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:30.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:30.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:30.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:30.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:30.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:30.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:30.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-20T21:28:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-20T21:28:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:30.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:30.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-20T21:28:30.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-20T21:28:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:30.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:30.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:30.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:28:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-20T21:28:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-20T21:28:30.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:30.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=49206' '!=' '' 2026-02-20T21:28:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:28:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=49206' = '' 2026-02-20T21:28:30.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:30.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:28:30.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:28:30.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:28:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:28:30.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:28:30.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:28:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:28:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:30.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:28:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:28:30.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=49206 2026-02-20T21:28:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 49206 2026-02-20T21:28:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:28:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:30.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:28:30.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-20T21:28:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-20T21:28:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-20T21:28:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-20T21:28:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-20T21:28:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:30.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-20T21:28:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:30.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-20T21:28:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:30.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-20T21:28:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-20T21:28:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:30.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:28:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-20T21:28:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-20T21:28:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:30.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:30.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:30.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=49206' '!=' 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:30.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:28:31.059 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:31.055+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e820 2026-02-20T21:28:31.059 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:31.055+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423c70 2026-02-20T21:28:31.060 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:31.056+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3055110 2026-02-20T21:28:31.063 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:31.059+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef2b0 2026-02-20T21:28:31.063 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:31.059+0000 7fd67aee6640 -1 reset not still connected to 0x55dab804ec30 2026-02-20T21:28:31.063 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:31.059+0000 7fd67aee6640 -1 reset not still connected to 0x55dab804f450 2026-02-20T21:28:31.063 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:31.059+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c08f0 2026-02-20T21:28:31.064 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:31.060+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-20T21:28:31.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-20T21:28:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-20T21:28:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-20T21:28:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:31.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:31.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-20T21:28:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:31.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:31.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:31.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:31.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:31.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:31.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:31.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-20T21:28:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-20T21:28:31.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:31.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:31.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:31.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:31.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:31.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:31.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-20T21:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-20T21:28:31.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:31.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:31.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:31.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:31.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:31.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:28:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-20T21:28:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-20T21:28:31.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:31.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:31.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:31.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:31.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=49206' '!=' 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:31.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:31.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:28:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-20T21:28:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-20T21:28:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-20T21:28:32.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-20T21:28:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:32.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:32.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:32.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-20T21:28:32.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:32.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:32.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:32.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:32.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:32.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:32.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:32.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:32.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:32.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:32.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-20T21:28:32.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:32.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:32.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-20T21:28:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:32.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:32.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:32.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:32.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:32.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:32.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:32.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-20T21:28:32.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:32.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:32.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-20T21:28:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:32.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:32.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:32.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:32.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:32.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:28:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-20T21:28:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:32.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-20T21:28:32.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:32.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:32.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:32.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:32.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:32.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:32.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:32.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:32.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:32.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:32.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:32.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:32.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:32.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:32.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:32.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:32.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:32.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:32.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=49206' '!=' 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:32.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:32.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:28:32.517 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:32.513+0000 7fd67aee6640 -1 reset not still connected to 0x55daafc932b0 2026-02-20T21:28:32.517 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:32.513+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1e820 2026-02-20T21:28:32.517 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:32.513+0000 7fd67aee6640 -1 reset not still connected to 0x55dab0423c70 2026-02-20T21:28:32.517 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:32.513+0000 7fd67aee6640 -1 reset not still connected to 0x55dab3440d00 2026-02-20T21:28:32.518 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:32.514+0000 7fd67aee6640 -1 reset not still connected to 0x55dab804ec30 2026-02-20T21:28:32.518 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:32.514+0000 7fd67aee6640 -1 reset not still connected to 0x55dab804f450 2026-02-20T21:28:32.518 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:32.514+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:33.288 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.284+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07acd00 2026-02-20T21:28:33.289 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.284+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b705b0 2026-02-20T21:28:33.290 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.286+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b712b0 2026-02-20T21:28:33.290 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.286+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c4e0 2026-02-20T21:28:33.290 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.286+0000 7fc9d03ed640 -1 reset not still connected to 0x557706d77930 2026-02-20T21:28:33.290 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.286+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0ea0 2026-02-20T21:28:33.290 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.286+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:28:33.694 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.690+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:33.694 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.690+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:33.694 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.690+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:33.694 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.690+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:33.695 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.690+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:33.695 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.690+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:33.695 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.690+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:33.695 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:33.690+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:34.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-20T21:28:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-20T21:28:34.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-20T21:28:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-20T21:28:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-20T21:28:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:34.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-20T21:28:34.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:34.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:34.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:34.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-20T21:28:34.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:34.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:34.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:34.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:34.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:34.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:34.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:34.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:34.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:34.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:34.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-20T21:28:34.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:34.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:34.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:34.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-20T21:28:34.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:34.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:34.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:34.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:34.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:34.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:34.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:34.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:34.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:34.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:34.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:28:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-20T21:28:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-20T21:28:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:34.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:34.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:34.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:34.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:34.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:34.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:34.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:34.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:34.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:34.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:34.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:34.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:34.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:34.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=49206' '!=' 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:28:34.571 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.567+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:34.571 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.567+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:34.572 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.567+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:34.572 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.567+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:34.573 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.569+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:34.573 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.569+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:34.573 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.569+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:34.573 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.569+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:34.888 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.884+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:34.888 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.884+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:34.888 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.884+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:34.888 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.884+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:34.888 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.884+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:34.889 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.884+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:34.889 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.885+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:34.890 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.885+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:34.893 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:34.889+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-20T21:28:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:36.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-20T21:28:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-20T21:28:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-20T21:28:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:36.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-20T21:28:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:36.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:36.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:36.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:36.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:36.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:36.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-20T21:28:36.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:36.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:36.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:36.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-20T21:28:36.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:36.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:36.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:36.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:36.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:36.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-20T21:28:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-20T21:28:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:36.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:36.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:36.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:36.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:36.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:36.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:36.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:36.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:36.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:36.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:36.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:28:36.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-20T21:28:36.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:36.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:36.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:36.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-20T21:28:36.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:36.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:36.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:36.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:36.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:36.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:36.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:36.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:36.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:36.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:36.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:36.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:36.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=49206' '!=' 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:36.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:36.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:28:37.696 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:37.692+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:37.697 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:37.692+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:37.765 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:37.761+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:37.765 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:37.761+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:37.808 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:37.803+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f07acd00 2026-02-20T21:28:37.809 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:37.803+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b705b0 2026-02-20T21:28:37.810 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:37.805+0000 7fc9d03ed640 -1 reset not still connected to 0x5576f2b712b0 2026-02-20T21:28:37.810 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:37.805+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4c4e0 2026-02-20T21:28:37.810 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:37.805+0000 7fc9d03ed640 -1 reset not still connected to 0x557706d77930 2026-02-20T21:28:37.811 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:37.806+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0ea0 2026-02-20T21:28:37.811 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:37.806+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1790 2026-02-20T21:28:38.343 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:38.339+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa09c0 2026-02-20T21:28:38.412 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:38.407+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:38.412 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:38.407+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:38.465 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:38.460+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:38.465 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:38.460+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-20T21:28:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-20T21:28:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-20T21:28:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-20T21:28:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:38.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:38.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-20T21:28:38.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:38.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:38.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:38.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:38.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:38.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:38.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:38.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:38.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:38.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:38.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:38.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:38.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:38.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-20T21:28:38.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:38.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:38.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:38.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-20T21:28:38.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:38.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:38.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:38.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:38.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:38.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:38.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:38.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:38.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:38.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:38.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:38.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:38.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:38.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-20T21:28:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-20T21:28:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:38.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:38.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:38.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:38.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:38.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:38.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:38.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:28:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-20T21:28:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-20T21:28:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:38.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:38.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:38.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=49206], [object_number=45, tag_tid=1, entry_tid=49205], [object_number=44, tag_tid=1, entry_tid=49204], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:38.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=49206' '!=' 'object_number=46, tag_tid=1, entry_tid=49206' 2026-02-20T21:28:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-20T21:28:39.405 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.401+0000 7fd67aee6640 -1 reset not still connected to 0x55daeca9e410 2026-02-20T21:28:39.748 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.744+0000 7fd67aee6640 -1 reset not still connected to 0x55dad05ab790 2026-02-20T21:28:39.749 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.744+0000 7fd67aee6640 -1 reset not still connected to 0x55dad12cef70 2026-02-20T21:28:39.751 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.747+0000 7fd67aee6640 -1 reset not still connected to 0x55dad12cf860 2026-02-20T21:28:39.751 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.747+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c1d40 2026-02-20T21:28:39.751 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.747+0000 7fd67aee6640 -1 reset not still connected to 0x55dad27db380 2026-02-20T21:28:39.751 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.747+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d380 2026-02-20T21:28:39.751 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.747+0000 7fd67aee6640 -1 reset not still connected to 0x55dae045d520 2026-02-20T21:28:39.751 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.747+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6c4e0 2026-02-20T21:28:39.751 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.747+0000 7fd67aee6640 -1 reset not still connected to 0x55daead268f0 2026-02-20T21:28:39.752 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.748+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27110 2026-02-20T21:28:39.942 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.938+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b860 2026-02-20T21:28:39.943 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.939+0000 7fd67aee6640 -1 reset not still connected to 0x55daead265b0 2026-02-20T21:28:39.943 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.939+0000 7fd67aee6640 -1 reset not still connected to 0x55daead272b0 2026-02-20T21:28:39.945 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:39.941+0000 7fd67aee6640 -1 reset not still connected to 0x55daeca9e410 2026-02-20T21:28:40.244 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.240+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:28:40.246 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.242+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d4c30 2026-02-20T21:28:40.246 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.242+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:28:40.247 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.242+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:40.247 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.243+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe410 2026-02-20T21:28:40.247 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.243+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806992b0 2026-02-20T21:28:40.248 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.243+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:40.248 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.243+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:40.486 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.482+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:28:40.487 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.482+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:28:40.487 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.483+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d4c30 2026-02-20T21:28:40.488 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.483+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:28:40.488 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.483+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:40.488 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.483+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe410 2026-02-20T21:28:40.488 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.483+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806992b0 2026-02-20T21:28:40.488 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.483+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:40.488 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.483+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:40.629 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.625+0000 7f413d8ae640 -1 reset not still connected to 0x55d280699860 2026-02-20T21:28:40.629 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.625+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b25b0 2026-02-20T21:28:40.631 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.627+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b35f0 2026-02-20T21:28:40.631 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.627+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c5b0 2026-02-20T21:28:40.631 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.627+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92bc70 2026-02-20T21:28:40.982 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.978+0000 7f413d8ae640 -1 reset not still connected to 0x55d270ea1a00 2026-02-20T21:28:40.983 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.978+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe9c0 2026-02-20T21:28:40.983 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.978+0000 7f413d8ae640 -1 reset not still connected to 0x55d280699860 2026-02-20T21:28:40.983 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.978+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b25b0 2026-02-20T21:28:40.983 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.978+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b35f0 2026-02-20T21:28:40.983 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.978+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0c5b0 2026-02-20T21:28:40.983 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:40.978+0000 7f413d8ae640 -1 reset not still connected to 0x55d29c92bc70 2026-02-20T21:28:41.092 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:41.088+0000 7f413d8ae640 -1 reset not still connected to 0x55d270385c70 2026-02-20T21:28:41.093 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:41.089+0000 7f413d8ae640 -1 reset not still connected to 0x55d2705be410 2026-02-20T21:28:41.093 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:41.089+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d4c30 2026-02-20T21:28:41.093 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:41.089+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5930 2026-02-20T21:28:41.093 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:41.089+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:41.093 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:41.090+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752fe410 2026-02-20T21:28:41.094 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:41.090+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806992b0 2026-02-20T21:28:41.094 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:41.090+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:41.096 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:41.092+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:41.252 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:41.247+0000 7fd67aee6640 -1 reset not still connected to 0x55daead265b0 2026-02-20T21:28:41.252 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:41.247+0000 7fd67aee6640 -1 reset not still connected to 0x55daead272b0 2026-02-20T21:28:41.252 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:41.247+0000 7fd67aee6640 -1 reset not still connected to 0x55daeca9e410 2026-02-20T21:28:41.725 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:41.719+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6c4e0 2026-02-20T21:28:41.725 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:41.720+0000 7fd67aee6640 -1 reset not still connected to 0x55daead268f0 2026-02-20T21:28:41.726 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:41.720+0000 7fd67aee6640 -1 reset not still connected to 0x55daead27110 2026-02-20T21:28:41.818 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.814+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:41.819 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.814+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:41.819 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.814+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:41.819 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.814+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:41.819 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.815+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:41.820 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.815+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:41.820 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.815+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:41.880 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.875+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:41.880 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.876+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:41.880 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.876+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:41.881 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.877+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:41.883 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.879+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:41.883 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.879+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:41.884 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.880+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:41.884 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.880+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:41.884 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.880+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:41.996 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.992+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e34f7a00 2026-02-20T21:28:41.996 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.992+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:41.998 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.994+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:41.998 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.994+0000 7fc9d03ed640 -1 reset not still connected to 0x5576ed276680 2026-02-20T21:28:41.998 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.994+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fa9a2dd0 2026-02-20T21:28:41.998 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.994+0000 7fc9d03ed640 -1 reset not still connected to 0x5576fe471e10 2026-02-20T21:28:42.000 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.996+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:42.000 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.996+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e0b60 2026-02-20T21:28:42.000 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:41.996+0000 7fc9d03ed640 -1 reset not still connected to 0x5577082e1520 2026-02-20T21:28:42.541 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:42.537+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e1fcb520 2026-02-20T21:28:42.542 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:42.537+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e3554750 2026-02-20T21:28:42.542 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:42.537+0000 7fc9d03ed640 -1 reset not still connected to 0x5576e35549c0 2026-02-20T21:28:42.542 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:28:42.539+0000 7fc9d03ed640 -1 reset not still connected to 0x557700d4d110 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_7 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_7 ']' 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_7' 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_7 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_7 2026-02-20T21:28:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:42.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:42.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_7'] 2026-02-20T21:28:42.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:42.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_7 2026-02-20T21:28:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:42.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:42.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_7 2026-02-20T21:28:42.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:42.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_7 2026-02-20T21:28:42.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:42.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:42.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '' 2026-02-20T21:28:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:42.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:42.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:42.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:28:42.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_7 2026-02-20T21:28:42.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:42.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:42.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:42.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_7 '..*' 2026-02-20T21:28:42.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:42.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:42.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:28:42.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:42.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_7.status 2026-02-20T21:28:42.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_7.status 2026-02-20T21:28:42.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:42.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:42.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_7 2026-02-20T21:28:42.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:42.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:28:42.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:28:42.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:42.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:42.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=49206], [object_number=47, tag_tid=1, entry_tid=49203]]], state=connected] 2026-02-20T21:28:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.status 2026-02-20T21:28:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:28:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:28:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_7 snap 2026-02-20T21:28:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:28:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:28:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:28:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:28:43.044 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:43.040+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbd40 2026-02-20T21:28:43.045 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:43.041+0000 7f413d8ae640 -1 reset not still connected to 0x55d280699520 2026-02-20T21:28:43.045 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:43.041+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:28:43.046 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:43.041+0000 7f413d8ae640 -1 reset not still connected to 0x55d293de6750 2026-02-20T21:28:43.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_7@snap 2026-02-20T21:28:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_7': 2026-02-20T21:28:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:28:43.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:28:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:28:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 119720767b63 2026-02-20T21:28:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.119720767b63 2026-02-20T21:28:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:28:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:28:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:28:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:28:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:20:06 2026 2026-02-20T21:28:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:20:06 2026 2026-02-20T21:28:43.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:21:29 2026 2026-02-20T21:28:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:28:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 119720767b63 2026-02-20T21:28:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:28:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:28:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 2d758829-29ac-41c3-9b79-87de526f77fa 2026-02-20T21:28:43.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:28:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:43.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_7 snap 2026-02-20T21:28:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:28:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:28:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:28:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.export 2026-02-20T21:28:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_7.export 2026-02-20T21:28:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_7.export 2026-02-20T21:28:43.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_7@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.export 2026-02-20T21:28:44.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:28:44.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_7@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_7.export 2026-02-20T21:28:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:28:44.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_7.export 2026-02-20T21:28:44.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_7.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_7.export 2026-02-20T21:28:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:28:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_8 2026-02-20T21:28:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_8 snap 2026-02-20T21:28:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:28:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:44.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:28:44.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_8@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:28:47.618 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:47.614+0000 7fd67aee6640 -1 reset not still connected to 0x55dab39ef2b0 2026-02-20T21:28:47.619 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:47.614+0000 7fd67aee6640 -1 reset not still connected to 0x55dab804ec30 2026-02-20T21:28:47.619 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:47.614+0000 7fd67aee6640 -1 reset not still connected to 0x55dab804f450 2026-02-20T21:28:47.619 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:47.614+0000 7fd67aee6640 -1 reset not still connected to 0x55dad14c08f0 2026-02-20T21:28:47.619 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:47.615+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_8 2026-02-20T21:28:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_8 started 2026-02-20T21:28:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:28:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:28:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:28:48.027 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:48.023+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_8 started 2026-02-20T21:28:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:28:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:28:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:28:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_8 2026-02-20T21:28:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:28:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:28:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:28:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:28:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:28:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:28:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_8 2026-02-20T21:28:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:28:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_8 2026-02-20T21:28:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:28:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:28:49.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_8 2026-02-20T21:28:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:28:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:49.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:28:49.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_8 2026-02-20T21:28:49.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:28:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:28:49.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_8 2026-02-20T21:28:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:28:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:49.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:28:49.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_8 2026-02-20T21:28:49.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:28:49.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:28:49.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:28:49.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:28:49.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:28:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:28:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:28:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_8 2026-02-20T21:28:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:28:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:28:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:28:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:28:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:28:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_8 2026-02-20T21:28:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:28:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:28:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:28:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:28:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:28:49.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:28:49.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:28:49.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:49.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:49.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:49.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:49.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:49.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:49.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:49.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:49.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:28:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:49.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:49.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:49.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:49.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:49.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:28:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:49.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_8 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:49.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:49.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:49.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:49.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:49.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:49.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:49.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:49.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-20T21:28:49.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:49.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:49.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:49.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-20T21:28:49.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:49.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:49.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:49.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:49.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-20T21:28:49.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-20T21:28:49.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-20T21:28:49.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:28:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:28:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35438], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35438], [object_number=33, tag_tid=1, entry_tid=35437], [object_number=32, tag_tid=1, entry_tid=35436], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:49.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:49.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:28:49.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-20T21:28:49.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:49.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:49.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:49.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-20T21:28:49.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:49.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:49.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:49.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:49.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-20T21:28:49.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-20T21:28:49.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-20T21:28:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:28:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:28:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35438], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:49.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35438], [object_number=33, tag_tid=1, entry_tid=35437], [object_number=32, tag_tid=1, entry_tid=35436], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:49.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35438' '!=' '' 2026-02-20T21:28:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:28:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35438' = '' 2026-02-20T21:28:49.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:49.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:28:49.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:28:49.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:49.661 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:49.657+0000 7fd67aee6640 -1 reset not still connected to 0x55daead265b0 2026-02-20T21:28:49.661 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:49.657+0000 7fd67aee6640 -1 reset not still connected to 0x55daead272b0 2026-02-20T21:28:49.661 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:49.658+0000 7fd67aee6640 -1 reset not still connected to 0x55daeca9e410 2026-02-20T21:28:49.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:28:49.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:49.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:28:49.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:49.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:28:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:28:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:28:49.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:49.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:28:49.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:49.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:28:49.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35438 2026-02-20T21:28:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 35438 2026-02-20T21:28:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:28:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:49.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:28:49.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-20T21:28:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-20T21:28:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-20T21:28:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-20T21:28:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:49.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-20T21:28:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:49.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:49.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:49.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:28:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-02-20T21:28:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:49.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:49.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:49.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:49.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:28:49.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:49.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:49.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:49.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_8 2026-02-20T21:28:49.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:49.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:49.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:49.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:49.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:49.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:49.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-20T21:28:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-20T21:28:50.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-20T21:28:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-20T21:28:50.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:50.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:50.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-20T21:28:50.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:28:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:28:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35438], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:50.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35438], [object_number=33, tag_tid=1, entry_tid=35437], [object_number=32, tag_tid=1, entry_tid=35436], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:50.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:28:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-20T21:28:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-20T21:28:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-20T21:28:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-20T21:28:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-20T21:28:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:28:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:28:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:50.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35438], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35438], [object_number=33, tag_tid=1, entry_tid=35437], [object_number=32, tag_tid=1, entry_tid=35436], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:50.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35438' '!=' 'object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:50.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:50.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:28:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-20T21:28:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-20T21:28:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-20T21:28:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-20T21:28:50.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:50.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:50.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:50.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-20T21:28:50.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:50.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:50.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:50.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:50.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:50.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:50.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:50.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:50.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:50.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:50.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:50.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:50.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:50.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:50.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:50.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:50.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:50.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:50.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:50.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:50.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:50.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:50.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:50.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:28:50.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:50.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:50.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:50.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-02-20T21:28:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:50.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:50.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:50.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:28:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:50.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_8 2026-02-20T21:28:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:50.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:50.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:50.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:50.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:50.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:50.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:50.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:50.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:50.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:50.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:50.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:50.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:50.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:50.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:50.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:50.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:50.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:50.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:50.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:50.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:50.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:50.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-20T21:28:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-20T21:28:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-20T21:28:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-20T21:28:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-20T21:28:50.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:50.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:28:50.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:28:50.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:50.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35438], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:50.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35438], [object_number=33, tag_tid=1, entry_tid=35437], [object_number=32, tag_tid=1, entry_tid=35436], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:50.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:50.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:28:50.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-20T21:28:50.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:50.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-20T21:28:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-20T21:28:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-20T21:28:50.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:50.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:50.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-20T21:28:50.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:50.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:28:50.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:28:50.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:50.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35438], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:50.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35438], [object_number=33, tag_tid=1, entry_tid=35437], [object_number=32, tag_tid=1, entry_tid=35436], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:50.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:50.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:50.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35438' '!=' 'object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:50.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:50.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:28:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-20T21:28:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-20T21:28:51.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-20T21:28:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-20T21:28:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:51.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:51.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-20T21:28:51.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:51.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:51.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:51.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:51.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:51.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:51.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:51.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:51.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:51.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:28:51.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:51.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-02-20T21:28:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:51.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:28:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_8 2026-02-20T21:28:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:51.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:51.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-20T21:28:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-20T21:28:51.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-20T21:28:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-20T21:28:51.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-20T21:28:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:28:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:28:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35438], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:51.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35438], [object_number=33, tag_tid=1, entry_tid=35437], [object_number=32, tag_tid=1, entry_tid=35436], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:51.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:51.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:28:51.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-20T21:28:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-20T21:28:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-20T21:28:51.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-20T21:28:51.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:51.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:51.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-20T21:28:51.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:28:51.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:28:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35438], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:51.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=34, tag_tid=1, entry_tid=35438], [object_number=33, tag_tid=1, entry_tid=35437], [object_number=32, tag_tid=1, entry_tid=35436], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:51.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:51.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:51.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=34, tag_tid=1, entry_tid=35438' '!=' 'object_number=34, tag_tid=1, entry_tid=35438' 2026-02-20T21:28:51.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:51.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:28:51.917 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:51.913+0000 7fd67aee6640 -1 reset not still connected to 0x55daead272b0 2026-02-20T21:28:51.917 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:51.913+0000 7fd67aee6640 -1 reset not still connected to 0x55daeca9e410 2026-02-20T21:28:52.860 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:52.855+0000 7f413d8ae640 -1 reset not still connected to 0x55d2735b9d40 2026-02-20T21:28:52.860 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:52.855+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743d5ad0 2026-02-20T21:28:52.860 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:52.855+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbd40 2026-02-20T21:28:52.860 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:52.856+0000 7f413d8ae640 -1 reset not still connected to 0x55d280699520 2026-02-20T21:28:52.860 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:52.856+0000 7f413d8ae640 -1 reset not still connected to 0x55d2837292b0 2026-02-20T21:28:52.860 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:52.856+0000 7f413d8ae640 -1 reset not still connected to 0x55d293de6750 2026-02-20T21:28:53.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_8 2026-02-20T21:28:53.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:53.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:53.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:53.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:53.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_8 ']' 2026-02-20T21:28:53.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_8' 2026-02-20T21:28:53.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_8 2026-02-20T21:28:53.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:53.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:53.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:53.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:53.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:53.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:53.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:53.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_8 2026-02-20T21:28:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:53.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:53.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:53.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:28:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:53.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:53.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_8 2026-02-20T21:28:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:53.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:53.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:53.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:53.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:53.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:53.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:53.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:53.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_8'] 2026-02-20T21:28:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:53.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_8 2026-02-20T21:28:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:53.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:53.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:53.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:53.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:53.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:53.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:53.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:53.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:53.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:53.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:53.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:53.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:53.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:53.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:53.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:53.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:53.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_8 2026-02-20T21:28:53.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:53.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_8 2026-02-20T21:28:53.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:53.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '' 2026-02-20T21:28:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-20T21:28:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-20T21:28:53.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:53.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:53.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-20T21:28:53.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:53.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:28:53.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:28:53.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:53.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35438], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:53.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35438], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:53.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:53.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:28:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_8 2026-02-20T21:28:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_8 '..*' 2026-02-20T21:28:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:28:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_8.status 2026-02-20T21:28:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_8.status 2026-02-20T21:28:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_8 2026-02-20T21:28:53.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:53.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:28:53.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:28:53.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:53.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35438], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:53.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=34, tag_tid=1, entry_tid=35438], [object_number=35, tag_tid=1, entry_tid=35435]]], state=connected] 2026-02-20T21:28:53.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.status 2026-02-20T21:28:53.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:28:53.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:28:53.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:53.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_8 snap 2026-02-20T21:28:53.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:53.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:53.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:53.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:28:53.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:28:53.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:28:53.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:28:54.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_8@snap 2026-02-20T21:28:54.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_8': 2026-02-20T21:28:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:28:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:28:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:28:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11b369550ae6 2026-02-20T21:28:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11b369550ae6 2026-02-20T21:28:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:28:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:28:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:28:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:28:54.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:20:18 2026 2026-02-20T21:28:54.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:20:18 2026 2026-02-20T21:28:54.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:21:19 2026 2026-02-20T21:28:54.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:28:54.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11b369550ae6 2026-02-20T21:28:54.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:28:54.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:28:54.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 6b084c95-95b7-4b13-a9e0-89f54b9d719a 2026-02-20T21:28:54.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:28:54.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_8 snap 2026-02-20T21:28:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:28:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:28:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:28:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.export 2026-02-20T21:28:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_8.export 2026-02-20T21:28:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_8.export 2026-02-20T21:28:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_8@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.export 2026-02-20T21:28:54.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:28:54.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_8@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_8.export 2026-02-20T21:28:54.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:28:54.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_8.export 2026-02-20T21:28:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_8.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_8.export 2026-02-20T21:28:55.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:55.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:28:55.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_9 2026-02-20T21:28:55.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_9 snap 2026-02-20T21:28:55.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:28:55.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:55.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:28:55.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:28:55.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_9@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:28:55.357 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:28:55.353+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:28:55.812 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:55.808+0000 7f413d8ae640 -1 reset not still connected to 0x55d2743fbe10 2026-02-20T21:28:55.813 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:55.809+0000 7f413d8ae640 -1 reset not still connected to 0x55d286a0cea0 2026-02-20T21:28:55.813 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:28:55.809+0000 7f413d8ae640 -1 reset not still connected to 0x55d28aa18000 2026-02-20T21:28:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_9 2026-02-20T21:28:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:28:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_9 started 2026-02-20T21:28:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:28:58.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:28:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:28:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:28:59.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_9 started 2026-02-20T21:28:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:28:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:28:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:28:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:28:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_9 2026-02-20T21:28:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:28:59.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:59.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:28:59.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:59.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:28:59.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:28:59.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:59.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:28:59.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:28:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_9 2026-02-20T21:28:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:28:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:28:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:28:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:28:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:28:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:28:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:28:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:28:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:28:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:28:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:28:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_9 2026-02-20T21:28:59.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:28:59.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:28:59.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:59.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:59.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_9 2026-02-20T21:28:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:28:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:28:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:28:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:28:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:28:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:28:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:28:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:28:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:28:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:28:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:28:59.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_9 2026-02-20T21:28:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:28:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:28:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_9 2026-02-20T21:28:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:28:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:28:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:28:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:28:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:28:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:28:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:28:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:28:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:28:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:28:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:28:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:28:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:28:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:28:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:28:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:59.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:59.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_9 2026-02-20T21:28:59.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:28:59.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:28:59.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:28:59.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:28:59.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:28:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:28:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:28:59.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_9 2026-02-20T21:28:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:28:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:28:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:28:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:28:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:28:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:28:59.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_9 2026-02-20T21:28:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:28:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:28:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:28:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:28:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:28:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:28:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:28:59.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:28:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:28:59.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_9 2026-02-20T21:28:59.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:59.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:28:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_9 ']' 2026-02-20T21:28:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-20T21:28:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-20T21:28:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:59.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:59.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:59.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:59.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-20T21:28:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:59.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:59.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-20T21:28:59.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:28:59.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:59.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-20T21:28:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:59.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:59.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:59.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:59.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:59.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:59.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-20T21:28:59.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:28:59.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:59.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:59.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-02-20T21:28:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:28:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:28:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:28:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:28:59.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:28:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:28:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:28:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:28:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:28:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:59.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:59.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:28:59.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:28:59.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:59.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:59.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:28:59.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:28:59.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:28:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:59.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:28:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-02-20T21:28:59.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:28:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-20T21:28:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:28:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-20T21:28:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:28:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:28:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-20T21:28:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-20T21:28:59.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:59.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:28:59.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-20T21:28:59.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:28:59.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:28:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:28:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:28:59.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=28637], [object_number=24, tag_tid=1, entry_tid=28636], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:28:59.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:28:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:28:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-20T21:28:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:28:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-20T21:28:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:28:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:28:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:28:59.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:28:59.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-20T21:28:59.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-20T21:28:59.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:28:59.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:28:59.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-20T21:28:59.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:28:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:28:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:28:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:28:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:28:59.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=28637], [object_number=24, tag_tid=1, entry_tid=28636], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:28:59.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:28:59.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:28:59.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:28:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=28637' '!=' '' 2026-02-20T21:28:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:28:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=28637' = '' 2026-02-20T21:28:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:28:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:28:59.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:28:59.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:59.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:28:59.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:28:59.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:28:59.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:59.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:28:59.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:28:59.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:28:59.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:59.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:28:59.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:28:59.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:28:59.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:28:59.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28637 2026-02-20T21:28:59.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 28637 2026-02-20T21:28:59.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:28:59.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:28:59.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:28:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_9 2026-02-20T21:28:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:28:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:28:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:28:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_9 ']' 2026-02-20T21:28:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-20T21:28:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-20T21:28:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:28:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:28:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:28:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:28:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:28:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:28:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:28:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:28:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-20T21:28:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:28:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:59.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:28:59.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:28:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:28:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:28:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:28:59.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:59.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:28:59.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:28:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:28:59.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-20T21:28:59.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:28:59.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:28:59.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:28:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-20T21:28:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:28:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:28:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:28:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:28:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:28:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:28:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:28:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:28:59.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:28:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:28:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:28:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:28:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:28:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:28:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:28:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:28:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:28:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:28:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:28:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:28:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:28:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:28:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:28:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:28:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:28:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:28:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:28:59.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-20T21:28:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:28:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:28:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-02-20T21:29:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:00.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:00.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:00.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-02-20T21:29:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-20T21:29:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:00.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-20T21:29:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-20T21:29:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-20T21:29:00.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:00.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:00.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-20T21:29:00.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:00.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:29:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:29:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=28637], [object_number=24, tag_tid=1, entry_tid=28636], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:00.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:29:00.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-20T21:29:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-20T21:29:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:00.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-20T21:29:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-20T21:29:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:00.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-20T21:29:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:29:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:29:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=28637], [object_number=24, tag_tid=1, entry_tid=28636], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:29:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:29:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=28637' '!=' 'object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:29:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:29:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_9 2026-02-20T21:29:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:00.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:29:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_9 ']' 2026-02-20T21:29:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-20T21:29:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-20T21:29:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:00.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:00.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:00.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:00.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:00.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-20T21:29:00.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-20T21:29:00.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:29:00.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:00.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-20T21:29:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:00.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:00.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:00.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:00.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-20T21:29:00.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:29:00.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:00.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-02-20T21:29:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:00.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:00.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:00.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-02-20T21:29:00.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:00.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-20T21:29:00.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:00.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:00.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:00.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-20T21:29:00.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-20T21:29:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-20T21:29:00.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:00.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:00.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-20T21:29:00.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:29:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:29:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=28637], [object_number=24, tag_tid=1, entry_tid=28636], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:00.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:29:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-20T21:29:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-20T21:29:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-20T21:29:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-20T21:29:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:00.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:00.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-20T21:29:00.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:00.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:29:00.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:29:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:00.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=28637], [object_number=24, tag_tid=1, entry_tid=28636], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:29:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:29:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=28637' '!=' 'object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:29:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:29:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_9 2026-02-20T21:29:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:29:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_9 ']' 2026-02-20T21:29:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-20T21:29:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-20T21:29:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:01.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:01.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:01.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:01.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:01.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:01.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:01.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:01.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-20T21:29:01.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:29:01.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:01.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-20T21:29:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:01.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:01.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:01.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:01.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-20T21:29:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:29:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:01.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-02-20T21:29:01.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:01.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-20T21:29:01.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:01.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:01.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-20T21:29:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-20T21:29:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-20T21:29:01.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-20T21:29:01.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:01.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:29:01.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:29:01.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:01.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:01.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=28637], [object_number=24, tag_tid=1, entry_tid=28636], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:01.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:01.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:29:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-20T21:29:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-20T21:29:01.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-20T21:29:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-20T21:29:01.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-20T21:29:01.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:01.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:29:01.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:29:01.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:01.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:01.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=28637], [object_number=24, tag_tid=1, entry_tid=28636], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:01.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:29:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:29:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=28637' '!=' 'object_number=25, tag_tid=1, entry_tid=28637' 2026-02-20T21:29:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:01.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:29:03.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_9 2026-02-20T21:29:03.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:03.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:03.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:29:03.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:03.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_9 ']' 2026-02-20T21:29:03.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_9' 2026-02-20T21:29:03.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_9 2026-02-20T21:29:03.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:03.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:03.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:03.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:03.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:03.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:03.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:03.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:03.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:03.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:03.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_9 2026-02-20T21:29:03.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:03.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:03.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:03.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:03.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:03.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:03.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:03.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_9 2026-02-20T21:29:03.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:29:03.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:03.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_9 2026-02-20T21:29:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:03.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:03.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:03.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:03.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_9 2026-02-20T21:29:03.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_9'] 2026-02-20T21:29:03.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:03.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:03.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:03.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_9 2026-02-20T21:29:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:03.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:03.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:03.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_9 2026-02-20T21:29:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_9 2026-02-20T21:29:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:03.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '' 2026-02-20T21:29:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-20T21:29:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-20T21:29:03.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:03.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:03.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-20T21:29:03.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:29:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:29:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:03.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:03.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:03.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:29:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_9 2026-02-20T21:29:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:03.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_9 '..*' 2026-02-20T21:29:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:29:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_9.status 2026-02-20T21:29:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_9.status 2026-02-20T21:29:03.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_9 2026-02-20T21:29:03.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:03.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:29:03.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:29:03.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:03.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:03.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28635], [object_number=26, tag_tid=1, entry_tid=28634]]], state=connected] 2026-02-20T21:29:03.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.status 2026-02-20T21:29:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:29:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:29:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_9 snap 2026-02-20T21:29:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:29:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:29:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:29:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:29:04.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_9@snap 2026-02-20T21:29:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_9': 2026-02-20T21:29:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:29:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:29:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:29:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ec57d7d900 2026-02-20T21:29:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ec57d7d900 2026-02-20T21:29:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:29:04.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:29:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:29:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:29:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:20:29 2026 2026-02-20T21:29:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:20:29 2026 2026-02-20T21:29:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:21:30 2026 2026-02-20T21:29:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:29:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ec57d7d900 2026-02-20T21:29:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:29:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:29:04.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 3b66b40e-1165-46a7-8a5e-c01a45d799ea 2026-02-20T21:29:04.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:29:04.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:04.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_9 snap 2026-02-20T21:29:04.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:04.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:29:04.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:04.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:29:04.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.export 2026-02-20T21:29:04.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_9.export 2026-02-20T21:29:04.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_9.export 2026-02-20T21:29:04.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_9@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.export 2026-02-20T21:29:04.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:29:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_9@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_9.export 2026-02-20T21:29:04.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:29:05.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_9.export 2026-02-20T21:29:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_9.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_9.export 2026-02-20T21:29:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:29:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_10 2026-02-20T21:29:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_10 snap 2026-02-20T21:29:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:29:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:05.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_10@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:29:05.599 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:05.595+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:29:06.773 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:29:06.770+0000 7f03c8435640 -1 reset not still connected to 0x55afb1e4e0d0 2026-02-20T21:29:06.773 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:29:06.770+0000 7f03c8435640 -1 reset not still connected to 0x55afb1e4e4e0 2026-02-20T21:29:06.773 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:29:06.770+0000 7f03c8435640 -1 reset not still connected to 0x55afd0b06d00 2026-02-20T21:29:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_10 2026-02-20T21:29:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:29:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_10 started 2026-02-20T21:29:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:29:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:29:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:29:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:29:07.489 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:07.485+0000 7fd67aee6640 -1 reset not still connected to 0x55dae3f6dee0 2026-02-20T21:29:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_10 started 2026-02-20T21:29:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:29:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:29:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:29:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:29:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_10 2026-02-20T21:29:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:29:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:29:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:29:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:29:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:29:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:29:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_10 2026-02-20T21:29:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:29:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:29:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:29:08.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:08.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:08.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_10 2026-02-20T21:29:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-20T21:29:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:08.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_10 2026-02-20T21:29:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:29:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:08.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:08.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:29:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:29:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:08.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:29:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:29:08.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:08.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_10 2026-02-20T21:29:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-20T21:29:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_10 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:29:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:29:08.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:08.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:08.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_10 2026-02-20T21:29:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:29:08.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:29:08.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:29:08.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:29:08.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:29:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:29:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:29:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_10 2026-02-20T21:29:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:29:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_10 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:29:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:29:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:08.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:29:08.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_10 2026-02-20T21:29:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:29:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_10 ']' 2026-02-20T21:29:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_10' 2026-02-20T21:29:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_10 2026-02-20T21:29:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:08.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:08.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_10 2026-02-20T21:29:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:08.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:08.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:08.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:08.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:08.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:08.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:08.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:08.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:08.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:08.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:08.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:08.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:08.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:08.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:08.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:08.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:08.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_10 2026-02-20T21:29:08.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-20T21:29:08.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:08.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_10 2026-02-20T21:29:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:08.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:08.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_10 2026-02-20T21:29:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_10'] 2026-02-20T21:29:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_10 2026-02-20T21:29:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:08.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_10 2026-02-20T21:29:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_10 2026-02-20T21:29:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-20T21:29:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_10 '' 2026-02-20T21:29:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-20T21:29:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-20T21:29:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-20T21:29:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_10.status 2026-02-20T21:29:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-20T21:29:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_10.status 2026-02-20T21:29:08.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:08.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:08.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:08.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=41991], [object_number=42, tag_tid=1, entry_tid=41990]]], state=connected] 2026-02-20T21:29:08.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=41991], [object_number=42, tag_tid=1, entry_tid=41990]]], state=connected] 2026-02-20T21:29:08.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_10.status 2026-02-20T21:29:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_10 2026-02-20T21:29:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-20T21:29:08.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_10 '..*' 2026-02-20T21:29:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-20T21:29:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_10.status 2026-02-20T21:29:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_10.status 2026-02-20T21:29:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:08.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_10.status 2026-02-20T21:29:08.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_10 2026-02-20T21:29:08.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_10.status 2026-02-20T21:29:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=41991], [object_number=42, tag_tid=1, entry_tid=41990]]], state=connected] 2026-02-20T21:29:08.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=41991], [object_number=42, tag_tid=1, entry_tid=41990]]], state=connected] 2026-02-20T21:29:08.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_10.status 2026-02-20T21:29:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_10 snap 2026-02-20T21:29:08.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:29:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:29:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:29:08.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:29:09.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_10@snap 2026-02-20T21:29:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_10': 2026-02-20T21:29:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:29:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:29:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:29:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ffc0bd83b7 2026-02-20T21:29:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ffc0bd83b7 2026-02-20T21:29:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:29:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:29:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:29:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:29:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:20:38 2026 2026-02-20T21:29:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:20:38 2026 2026-02-20T21:29:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:21:39 2026 2026-02-20T21:29:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:29:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ffc0bd83b7 2026-02-20T21:29:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:29:09.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:29:09.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: f6afe94a-4ead-4dcd-8461-806c99203a6b 2026-02-20T21:29:09.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:29:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_10 snap 2026-02-20T21:29:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:29:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:29:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_10.export 2026-02-20T21:29:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_10.export 2026-02-20T21:29:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_10.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_10.export 2026-02-20T21:29:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_10@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_10.export 2026-02-20T21:29:09.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:29:09.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_10@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_10.export 2026-02-20T21:29:09.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:29:09.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_10.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_10.export 2026-02-20T21:29:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_10.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_10.export 2026-02-20T21:29:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:29:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_11 2026-02-20T21:29:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_11 snap 2026-02-20T21:29:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_11@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:29:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_11 2026-02-20T21:29:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_11 started 2026-02-20T21:29:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:13.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:29:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:29:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:13.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:29:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_11 started 2026-02-20T21:29:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:29:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:29:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:29:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_11 2026-02-20T21:29:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:29:14.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:29:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:29:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:29:14.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:29:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:29:14.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_11 2026-02-20T21:29:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:29:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:14.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:29:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:29:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:29:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:29:14.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:14.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:14.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_11 2026-02-20T21:29:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:14.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:14.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_11 2026-02-20T21:29:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:29:14.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:14.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:29:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:29:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:14.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:29:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:29:14.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:14.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:14.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_11 2026-02-20T21:29:14.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:14.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:14.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_11 2026-02-20T21:29:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:29:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:29:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:29:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:29:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:14.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:14.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_11 2026-02-20T21:29:14.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:14.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:14.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:14.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:14.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_11 2026-02-20T21:29:14.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:29:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:14.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:14.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:29:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:29:14.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:14.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:14.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:14.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:14.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_11 2026-02-20T21:29:14.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:29:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:29:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:29:14.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:29:14.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:29:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:29:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:29:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_11 2026-02-20T21:29:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:29:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:29:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:29:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:14.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:29:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_11 2026-02-20T21:29:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:29:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:29:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:29:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:29:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:29:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:29:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:14.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:29:14.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_11 2026-02-20T21:29:14.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:14.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:14.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:14.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:14.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_11 ']' 2026-02-20T21:29:14.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-20T21:29:14.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-20T21:29:14.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:14.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:14.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:14.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:14.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:14.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:14.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:14.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:14.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:14.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-20T21:29:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:14.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:14.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:14.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:14.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:14.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:14.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:14.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:14.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-20T21:29:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:14.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:14.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:14.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:14.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:14.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:14.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:14.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:14.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-02-20T21:29:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:14.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:14.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:14.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-02-20T21:29:14.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:14.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:14.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:14.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:14.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:14.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:14.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:14.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:14.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:14.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:14.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:14.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-20T21:29:14.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:14.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:14.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:14.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-20T21:29:14.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:14.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:14.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:14.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:14.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:29:14.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-20T21:29:14.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:14.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:14.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-20T21:29:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:29:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:29:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:14.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30573], [object_number=28, tag_tid=1, entry_tid=30572], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:29:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-20T21:29:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:14.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:14.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-20T21:29:14.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:14.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:14.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:14.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:14.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:29:14.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-20T21:29:14.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:14.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:14.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-20T21:29:14.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:14.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:29:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:29:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:14.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30573], [object_number=28, tag_tid=1, entry_tid=30572], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:14.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=30573' '!=' '' 2026-02-20T21:29:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:29:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=30573' = '' 2026-02-20T21:29:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:29:14.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:29:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:14.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:29:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:29:14.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:14.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:29:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:29:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:29:14.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:29:14.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:29:14.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=30573 2026-02-20T21:29:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 30573 2026-02-20T21:29:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:29:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:14.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:29:15.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_11 2026-02-20T21:29:15.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:15.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_11 ']' 2026-02-20T21:29:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-20T21:29:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-20T21:29:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:15.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:15.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-20T21:29:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:15.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:15.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:15.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:15.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:15.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:15.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-20T21:29:15.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:15.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:15.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:15.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:15.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:15.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:15.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-02-20T21:29:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:15.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:15.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:15.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:15.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:15.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:15.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-02-20T21:29:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:15.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:15.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:15.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:15.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-20T21:29:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-20T21:29:15.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:29:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-20T21:29:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-20T21:29:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:29:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:29:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30573], [object_number=28, tag_tid=1, entry_tid=30572], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:29:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-20T21:29:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-20T21:29:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:15.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:15.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:15.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:15.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:29:15.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-20T21:29:15.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-20T21:29:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:15.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:29:15.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:29:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:15.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30573], [object_number=28, tag_tid=1, entry_tid=30572], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:15.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:15.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=30573' '!=' 'object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:15.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:29:15.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_11 2026-02-20T21:29:15.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:15.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:15.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:15.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:15.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_11 ']' 2026-02-20T21:29:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-20T21:29:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-20T21:29:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:15.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:15.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:15.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-20T21:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:15.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:15.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:15.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:15.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:15.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:15.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:15.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:15.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:15.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:15.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-20T21:29:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:15.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:15.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:15.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:15.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:15.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-02-20T21:29:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:15.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:15.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:15.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:15.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-02-20T21:29:15.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:15.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:15.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:15.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:15.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-20T21:29:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-20T21:29:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:29:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-20T21:29:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-20T21:29:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:15.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:29:15.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:29:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:15.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30573], [object_number=28, tag_tid=1, entry_tid=30572], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:15.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:29:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-20T21:29:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-20T21:29:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:29:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-20T21:29:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-20T21:29:15.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:29:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:29:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:16.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30573], [object_number=28, tag_tid=1, entry_tid=30572], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:16.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=30573' '!=' 'object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:16.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:29:16.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_11 2026-02-20T21:29:16.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:16.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_11 ']' 2026-02-20T21:29:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-20T21:29:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-20T21:29:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-20T21:29:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:16.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:16.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:16.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:16.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:16.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:16.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:16.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:16.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:16.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-20T21:29:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:16.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:16.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:16.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:16.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:16.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-02-20T21:29:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:16.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:16.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:16.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:16.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:16.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:16.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-02-20T21:29:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:16.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:16.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:17.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-20T21:29:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-20T21:29:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:29:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-20T21:29:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-20T21:29:17.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:17.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:29:17.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:29:17.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:17.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:17.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30573], [object_number=28, tag_tid=1, entry_tid=30572], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:17.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:29:17.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-20T21:29:17.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:17.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-20T21:29:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:29:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-20T21:29:17.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-20T21:29:17.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:29:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:29:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30573], [object_number=28, tag_tid=1, entry_tid=30572], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=30573' '!=' 'object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:29:18.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_11 2026-02-20T21:29:18.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:18.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_11 ']' 2026-02-20T21:29:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-20T21:29:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-20T21:29:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:18.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:18.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-20T21:29:18.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:18.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:18.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:18.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:18.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:18.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:18.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:18.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:18.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:18.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:18.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:18.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-20T21:29:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:18.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:18.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:18.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:18.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:18.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-02-20T21:29:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:18.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:18.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:18.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:18.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:18.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:18.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:18.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:18.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:18.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-02-20T21:29:18.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:18.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:18.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:18.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:18.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:18.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:18.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:18.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:18.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:18.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:18.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:18.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-20T21:29:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-20T21:29:18.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:29:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-20T21:29:18.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-20T21:29:18.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:18.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:29:18.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:29:18.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:18.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:18.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30573], [object_number=28, tag_tid=1, entry_tid=30572], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:18.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:18.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:29:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-20T21:29:18.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-20T21:29:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:29:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-20T21:29:18.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-20T21:29:18.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:29:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:29:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:19.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=1, entry_tid=30573], [object_number=28, tag_tid=1, entry_tid=30572], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:19.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:19.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:19.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:19.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=29, tag_tid=1, entry_tid=30573' '!=' 'object_number=29, tag_tid=1, entry_tid=30573' 2026-02-20T21:29:19.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:19.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:29:21.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_11 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_11 ']' 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_11' 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_11 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:21.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_11 2026-02-20T21:29:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:21.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:21.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:21.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:21.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:21.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:21.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_11 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:21.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:21.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:21.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:21.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:21.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_11 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:21.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:21.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:21.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:21.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:21.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:21.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:21.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:21.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:21.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:21.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_11'] 2026-02-20T21:29:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:21.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:21.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:21.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_11 2026-02-20T21:29:21.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:21.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:21.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:21.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:21.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:21.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:21.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:21.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:21.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:21.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_11 2026-02-20T21:29:21.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_11 2026-02-20T21:29:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '' 2026-02-20T21:29:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:29:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-20T21:29:21.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:21.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-20T21:29:21.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:29:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:29:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:21.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:21.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:29:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_11 2026-02-20T21:29:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_11 '..*' 2026-02-20T21:29:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:29:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:21.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_11.status 2026-02-20T21:29:21.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_11.status 2026-02-20T21:29:21.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_11 2026-02-20T21:29:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:21.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:29:21.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:29:21.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:21.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:21.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=30571], [object_number=30, tag_tid=1, entry_tid=30570]]], state=connected] 2026-02-20T21:29:21.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.status 2026-02-20T21:29:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:29:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:29:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_11 snap 2026-02-20T21:29:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:29:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:29:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:29:22.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_11@snap 2026-02-20T21:29:22.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_11': 2026-02-20T21:29:22.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:29:22.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:29:22.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:29:22.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11978c22887d 2026-02-20T21:29:22.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11978c22887d 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:20:49 2026 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:20:49 2026 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:21:50 2026 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11978c22887d 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 1b56fc83-a4ca-4bf2-9ad3-98d355f51e76 2026-02-20T21:29:22.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:29:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_11 snap 2026-02-20T21:29:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:29:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:29:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.export 2026-02-20T21:29:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_11.export 2026-02-20T21:29:22.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_11.export 2026-02-20T21:29:22.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_11@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.export 2026-02-20T21:29:22.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:29:22.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_11@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_11.export 2026-02-20T21:29:22.511 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:22.507+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:29:22.511 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:22.507+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:29:22.512 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:22.508+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:29:22.512 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:22.508+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:29:22.513 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:22.509+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:29:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:29:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_11.export 2026-02-20T21:29:22.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_11.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_11.export 2026-02-20T21:29:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:29:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_12 2026-02-20T21:29:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_12 snap 2026-02-20T21:29:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_12@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:29:25.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_12 2026-02-20T21:29:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_12 started 2026-02-20T21:29:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:29:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:29:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:29:25.764 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.8204478497244713) > weight (0.25) 2026-02-20T21:29:25.764 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.28027760603757135) > weight (0.25) 2026-02-20T21:29:25.765 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.6410988207954768) > weight (0.25) 2026-02-20T21:29:25.765 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.35324159569053293) > weight (0.25) 2026-02-20T21:29:25.765 INFO:tasks.rbd_mirror_thrash:waiting for 119.8425341855503 secs before thrashing 2026-02-20T21:29:26.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_12 started 2026-02-20T21:29:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:29:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:29:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:29:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_12 2026-02-20T21:29:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:29:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:29:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:29:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:29:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:29:26.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:29:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_12 2026-02-20T21:29:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:29:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:26.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:26.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:29:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:29:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:26.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:26.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:29:26.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:29:26.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:26.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:26.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_12 2026-02-20T21:29:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:26.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:26.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_12 2026-02-20T21:29:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:29:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:26.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:26.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:29:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:29:26.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:26.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_12 2026-02-20T21:29:26.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:26.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_12 2026-02-20T21:29:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:29:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:26.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:29:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:26.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:29:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:29:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:29:26.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:26.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:26.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:26.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:26.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_12 2026-02-20T21:29:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:29:26.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:29:26.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:29:26.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:29:26.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:29:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:29:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:29:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_12 2026-02-20T21:29:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:29:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:29:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:29:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:29:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_12 2026-02-20T21:29:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:29:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:29:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:29:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:29:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:29:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:29:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:29:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_12 2026-02-20T21:29:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_12 ']' 2026-02-20T21:29:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-20T21:29:26.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-20T21:29:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:26.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-20T21:29:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:26.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:26.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:26.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:26.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:26.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:26.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:26.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:26.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2026-02-20T21:29:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:26.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:26.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:26.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:26.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:26.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:26.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:26.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:26.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2026-02-20T21:29:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:26.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:26.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:26.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:26.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:26.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-20T21:29:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-20T21:29:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:26.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:26.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:26.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:26.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:26.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:26.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:26.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:26.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:26.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:26.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:26.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:26.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:26.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47946], [object_number=45, tag_tid=1, entry_tid=47945], [object_number=44, tag_tid=1, entry_tid=47944], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:26.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:26.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:29:26.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-20T21:29:26.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:26.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:26.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:26.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-20T21:29:26.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:26.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:26.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:26.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:26.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:26.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:26.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:26.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:26.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:26.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:26.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:26.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:26.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47946], [object_number=45, tag_tid=1, entry_tid=47945], [object_number=44, tag_tid=1, entry_tid=47944], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:26.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47946' '!=' '' 2026-02-20T21:29:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:29:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47946' = '' 2026-02-20T21:29:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:29:26.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:29:26.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:26.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:29:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:29:26.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:26.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:29:26.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:29:26.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:29:26.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:26.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:29:26.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:29:26.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:26.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=47946 2026-02-20T21:29:26.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 47946 2026-02-20T21:29:26.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:29:26.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:26.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:29:26.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_12 2026-02-20T21:29:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_12 ']' 2026-02-20T21:29:26.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-20T21:29:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-20T21:29:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:26.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:26.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:26.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-20T21:29:26.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:26.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2026-02-20T21:29:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:26.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:26.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:26.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:26.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:27.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:27.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:27.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:27.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:27.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2026-02-20T21:29:27.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:27.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:27.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:27.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:27.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:27.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:27.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:27.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:27.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:27.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:27.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:27.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:27.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:27.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:27.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-20T21:29:27.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:27.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:27.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-20T21:29:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:27.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:27.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:27.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47946], [object_number=45, tag_tid=1, entry_tid=47945], [object_number=44, tag_tid=1, entry_tid=47944], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:27.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:27.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:29:27.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-20T21:29:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-20T21:29:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:27.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:27.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:27.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:27.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:27.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47946], [object_number=45, tag_tid=1, entry_tid=47945], [object_number=44, tag_tid=1, entry_tid=47944], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:27.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:27.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:27.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:27.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47946' '!=' 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:27.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:27.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:29:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_12 2026-02-20T21:29:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:27.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_12 ']' 2026-02-20T21:29:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-20T21:29:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-20T21:29:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:27.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:27.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-20T21:29:27.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:27.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:27.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:27.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:27.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:27.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:27.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2026-02-20T21:29:27.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:27.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:27.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:27.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:27.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2026-02-20T21:29:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:27.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:27.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:27.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:27.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:27.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:27.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:27.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:27.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-20T21:29:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-20T21:29:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:27.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:27.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:27.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:27.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47946], [object_number=45, tag_tid=1, entry_tid=47945], [object_number=44, tag_tid=1, entry_tid=47944], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:27.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:27.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:29:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-20T21:29:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:27.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:27.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:27.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-20T21:29:27.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:27.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:27.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:27.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:27.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:27.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:27.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:27.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:27.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:27.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:27.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:27.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47946], [object_number=45, tag_tid=1, entry_tid=47945], [object_number=44, tag_tid=1, entry_tid=47944], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:27.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:27.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:27.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:27.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47946' '!=' 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:27.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:27.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_12 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_12 ']' 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:28.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:28.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:28.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:28.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:28.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:28.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:28.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:28.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:28.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:28.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:28.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:28.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:28.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:28.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-20T21:29:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-20T21:29:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:28.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:28.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:28.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:28.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47946], [object_number=45, tag_tid=1, entry_tid=47945], [object_number=44, tag_tid=1, entry_tid=47944], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:28.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:29:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-20T21:29:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-20T21:29:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:28.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:28.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:28.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:28.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:28.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47946], [object_number=45, tag_tid=1, entry_tid=47945], [object_number=44, tag_tid=1, entry_tid=47944], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:28.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:28.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:28.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:28.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47946' '!=' 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:28.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:28.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:29:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_12 2026-02-20T21:29:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_12 ']' 2026-02-20T21:29:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-20T21:29:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-20T21:29:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-20T21:29:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:30.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:30.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:30.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:30.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:30.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:30.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:30.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:30.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:30.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2026-02-20T21:29:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:30.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:30.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:30.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:30.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:30.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:30.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:30.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:30.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:30.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:30.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:30.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:30.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-20T21:29:30.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:30.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:30.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:30.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-20T21:29:30.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:30.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:30.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:30.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:30.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:30.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:30.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:30.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:30.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:30.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:30.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:30.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:30.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:30.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:30.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47946], [object_number=45, tag_tid=1, entry_tid=47945], [object_number=44, tag_tid=1, entry_tid=47944], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:30.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:30.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:29:30.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-20T21:29:30.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:30.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:30.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:30.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-20T21:29:30.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:30.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:30.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:30.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:30.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:30.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:30.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:30.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:30.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:30.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:30.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:30.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47946], [object_number=45, tag_tid=1, entry_tid=47945], [object_number=44, tag_tid=1, entry_tid=47944], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:30.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:30.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47946' '!=' 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:29:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_12 2026-02-20T21:29:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_12 ']' 2026-02-20T21:29:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-20T21:29:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-20T21:29:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-20T21:29:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:32.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:32.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:32.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:32.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:32.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:32.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:32.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2026-02-20T21:29:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:32.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:32.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:32.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:32.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:32.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:32.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:32.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:32.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:32.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:32.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:32.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:32.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:32.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:32.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2026-02-20T21:29:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:32.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:32.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:32.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:32.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:32.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:32.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:32.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:32.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:32.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:32.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:32.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:32.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:32.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-20T21:29:32.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:32.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-20T21:29:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:32.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:32.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:32.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:32.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:32.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:32.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:32.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:32.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47946], [object_number=45, tag_tid=1, entry_tid=47945], [object_number=44, tag_tid=1, entry_tid=47944], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:29:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-20T21:29:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-20T21:29:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:32.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:32.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=46, tag_tid=1, entry_tid=47946], [object_number=45, tag_tid=1, entry_tid=47945], [object_number=44, tag_tid=1, entry_tid=47944], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=46, tag_tid=1, entry_tid=47946' '!=' 'object_number=46, tag_tid=1, entry_tid=47946' 2026-02-20T21:29:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:29:34.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_12 2026-02-20T21:29:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_12 ']' 2026-02-20T21:29:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_12' 2026-02-20T21:29:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_12 2026-02-20T21:29:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:34.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:34.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_12 2026-02-20T21:29:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:34.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:34.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:34.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:34.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:34.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:34.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_12 2026-02-20T21:29:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:34.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:34.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:35.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:35.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:35.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:35.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_12'] 2026-02-20T21:29:35.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:35.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_12 2026-02-20T21:29:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:35.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:35.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_12 2026-02-20T21:29:35.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_12 2026-02-20T21:29:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '' 2026-02-20T21:29:35.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:35.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:35.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:35.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:35.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:35.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:29:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_12 2026-02-20T21:29:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_12 '..*' 2026-02-20T21:29:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:29:35.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:35.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_12.status 2026-02-20T21:29:35.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_12.status 2026-02-20T21:29:35.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:35.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:35.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_12 2026-02-20T21:29:35.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:29:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:29:35.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:35.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=46, tag_tid=1, entry_tid=47946], [object_number=47, tag_tid=1, entry_tid=47943]]], state=connected] 2026-02-20T21:29:35.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.status 2026-02-20T21:29:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:29:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:29:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_12 snap 2026-02-20T21:29:35.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:29:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:29:35.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:29:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_12@snap 2026-02-20T21:29:36.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_12': 2026-02-20T21:29:36.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:29:36.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:29:36.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:29:36.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11b34bc3ee1f 2026-02-20T21:29:36.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11b34bc3ee1f 2026-02-20T21:29:36.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:29:36.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:29:36.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:29:36.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:29:36.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:20:59 2026 2026-02-20T21:29:36.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:20:59 2026 2026-02-20T21:29:36.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:22:00 2026 2026-02-20T21:29:36.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:29:36.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11b34bc3ee1f 2026-02-20T21:29:36.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:29:36.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:29:36.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0c650e0a-5572-4f7c-bfc3-9c6d193b9e56 2026-02-20T21:29:36.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:29:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_12 snap 2026-02-20T21:29:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:36.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:29:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:29:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.export 2026-02-20T21:29:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_12.export 2026-02-20T21:29:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_12.export 2026-02-20T21:29:36.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_12@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.export 2026-02-20T21:29:36.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:29:36.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_12@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_12.export 2026-02-20T21:29:36.374 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:36.370+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:29:36.374 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:36.370+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:29:36.374 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:36.370+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:29:36.375 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:36.370+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:29:36.375 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:36.370+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:29:36.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:29:36.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_12.export 2026-02-20T21:29:36.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_12.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_12.export 2026-02-20T21:29:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:29:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_13 2026-02-20T21:29:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_13 snap 2026-02-20T21:29:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_13@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:29:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_13 2026-02-20T21:29:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_13 started 2026-02-20T21:29:38.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:29:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:29:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:38.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:29:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_13 started 2026-02-20T21:29:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:29:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:29:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:29:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_13 2026-02-20T21:29:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:29:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:29:39.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:29:39.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:29:39.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:39.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:29:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:29:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_13 2026-02-20T21:29:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:29:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:39.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:39.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:39.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:29:39.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:39.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:39.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:39.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:39.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:39.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:39.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:39.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:29:39.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:29:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:39.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:39.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:39.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:39.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_13 2026-02-20T21:29:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_13 2026-02-20T21:29:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:29:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:29:39.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:39.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:39.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_13 2026-02-20T21:29:39.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:39.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:39.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:39.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_13 2026-02-20T21:29:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:29:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:39.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:29:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:29:39.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:29:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:29:39.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:39.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:39.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_13 2026-02-20T21:29:39.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:39.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:39.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:39.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_13 2026-02-20T21:29:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:29:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:39.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:29:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:39.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:29:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:29:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:29:39.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:39.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:39.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_13 2026-02-20T21:29:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:29:39.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:29:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:29:39.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:29:39.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:29:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:29:39.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:29:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_13 2026-02-20T21:29:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:29:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:29:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:29:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:29:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_13 2026-02-20T21:29:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:29:39.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:29:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:29:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:29:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:29:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:29:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:39.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:29:39.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-20T21:29:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-20T21:29:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-20T21:29:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-20T21:29:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:39.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:39.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:39.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:39.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-20T21:29:39.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:39.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:39.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:39.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:39.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:39.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:39.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:39.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:39.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:39.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-20T21:29:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:39.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:39.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:39.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:39.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:39.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:39.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:39.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:39.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:39.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:39.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_13 2026-02-20T21:29:39.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:39.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:39.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:39.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:39.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:39.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:39.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:39.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:39.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:39.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:39.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:39.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:39.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:39.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:39.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:39.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:39.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:39.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:39.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:39.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:39.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:39.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:39.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:39.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:39.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:39.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:39.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:39.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:39.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_13 2026-02-20T21:29:39.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:39.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:39.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:39.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-20T21:29:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-20T21:29:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:29:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-20T21:29:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-20T21:29:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:39.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:29:39.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:29:39.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:39.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:39.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=37476], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474], [object_number=33, tag_tid=1, entry_tid=37473]]], state=connected] 2026-02-20T21:29:39.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:39.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:29:39.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-20T21:29:39.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-20T21:29:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:29:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-20T21:29:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:39.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:39.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-20T21:29:39.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:29:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:29:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:39.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=37476], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474], [object_number=33, tag_tid=1, entry_tid=37473]]], state=connected] 2026-02-20T21:29:39.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=37476' '!=' '' 2026-02-20T21:29:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:29:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=37476' = '' 2026-02-20T21:29:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:29:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:29:39.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:39.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:29:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:29:39.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:39.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:29:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:29:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:29:39.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:29:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:29:39.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=37476 2026-02-20T21:29:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 37476 2026-02-20T21:29:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:29:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:39.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:29:40.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-20T21:29:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-20T21:29:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-20T21:29:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-20T21:29:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:40.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:40.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-20T21:29:40.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:40.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:40.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:40.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:40.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:40.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:40.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-20T21:29:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:40.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:40.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:40.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_13 2026-02-20T21:29:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:40.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:40.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:40.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:40.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:40.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_13 2026-02-20T21:29:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:40.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:40.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:40.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:40.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:40.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:40.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:40.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:40.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:40.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:40.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-20T21:29:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:40.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-20T21:29:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:29:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-20T21:29:40.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-20T21:29:40.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:29:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:29:40.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:40.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:40.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=37476], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474], [object_number=33, tag_tid=1, entry_tid=37473]]], state=connected] 2026-02-20T21:29:40.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:40.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:29:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-20T21:29:40.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-20T21:29:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:29:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-20T21:29:40.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-20T21:29:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:29:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:29:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=37476], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474], [object_number=33, tag_tid=1, entry_tid=37473]]], state=connected] 2026-02-20T21:29:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:40.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:40.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:40.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=37476' '!=' 'object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:40.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:40.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:29:40.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-20T21:29:40.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:40.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:40.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:40.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:40.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-20T21:29:40.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-20T21:29:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-20T21:29:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:40.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:40.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-20T21:29:40.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:40.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:40.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:40.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:40.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:40.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:40.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:40.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:40.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:40.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-20T21:29:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:40.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:40.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:40.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:40.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:40.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:40.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:40.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:40.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:40.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:40.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:40.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:40.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:40.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:40.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:40.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:41.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:41.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:41.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:41.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:41.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_13 2026-02-20T21:29:41.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:41.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:41.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:41.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:41.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:41.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_13 2026-02-20T21:29:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:41.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:41.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:41.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:41.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-20T21:29:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-20T21:29:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:41.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:41.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:41.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:29:41.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-20T21:29:41.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:41.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:41.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-20T21:29:41.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:29:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:29:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=37476], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474], [object_number=33, tag_tid=1, entry_tid=37473]]], state=connected] 2026-02-20T21:29:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:41.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:29:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-20T21:29:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-20T21:29:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:41.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:29:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-20T21:29:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:41.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-20T21:29:41.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:41.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:29:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:29:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:41.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=37476], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474], [object_number=33, tag_tid=1, entry_tid=37473]]], state=connected] 2026-02-20T21:29:41.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:41.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:41.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=37476' '!=' 'object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:29:41.310 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-20T21:29:41.311 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-20T21:29:41.311 INFO:tasks.rbd_mirror_thrash:waiting for 0.7827153150946531 secs before reviving daemons 2026-02-20T21:29:41.312 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:29:41.309+0000 7fa430442640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 93321) UID: 1000 2026-02-20T21:29:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-20T21:29:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-20T21:29:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-20T21:29:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-20T21:29:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-20T21:29:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:42.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:42.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:42.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:42.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-20T21:29:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:42.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:42.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:42.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:42.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:42.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:42.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:42.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:42.095 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-20T21:29:42.095 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:29:42.095 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-20T21:29:42.096 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T21:29:42.096 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T21:29:42.098 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T21:29:42.098 INFO:tasks.rbd_mirror_thrash:waiting for 70.15366533568894 secs before thrashing 2026-02-20T21:29:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_13 2026-02-20T21:29:42.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:42.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:42.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:42.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:42.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:42.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:42.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:42.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:42.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:42.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_13 2026-02-20T21:29:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:42.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:42.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:42.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:42.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:42.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:42.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-20T21:29:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-20T21:29:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:42.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:29:42.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-20T21:29:42.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-20T21:29:42.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:29:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:29:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=37476], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474], [object_number=33, tag_tid=1, entry_tid=37473]]], state=connected] 2026-02-20T21:29:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:29:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-20T21:29:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-20T21:29:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:29:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-20T21:29:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-20T21:29:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:42.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:29:42.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:29:42.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:42.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=37476], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474], [object_number=33, tag_tid=1, entry_tid=37473]]], state=connected] 2026-02-20T21:29:42.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:42.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:42.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:42.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=37476' '!=' 'object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:42.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:42.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:29:44.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-20T21:29:44.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:44.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:44.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:44.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:44.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-20T21:29:44.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-20T21:29:44.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-20T21:29:44.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:44.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:44.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:44.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:44.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:44.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:44.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:44.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-20T21:29:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:44.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:44.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:44.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:44.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:44.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:44.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:44.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:44.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:44.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:44.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-20T21:29:44.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:44.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:44.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:44.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:44.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:44.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:44.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:44.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:44.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:44.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:44.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:44.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_13 2026-02-20T21:29:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:44.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:44.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:44.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:44.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:44.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:44.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_13 2026-02-20T21:29:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:44.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:44.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:44.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:44.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:44.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:44.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-20T21:29:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-20T21:29:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:29:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-20T21:29:44.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-20T21:29:44.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:44.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:29:44.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:29:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:44.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=37476], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474], [object_number=33, tag_tid=1, entry_tid=37473]]], state=connected] 2026-02-20T21:29:44.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:44.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:29:44.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-20T21:29:44.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-20T21:29:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:29:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-20T21:29:44.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-20T21:29:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:29:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:29:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=37476], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474], [object_number=33, tag_tid=1, entry_tid=37473]]], state=connected] 2026-02-20T21:29:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=37476' '!=' 'object_number=32, tag_tid=1, entry_tid=37476' 2026-02-20T21:29:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:44.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:29:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_13 2026-02-20T21:29:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:46.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_13 ']' 2026-02-20T21:29:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_13' 2026-02-20T21:29:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_13 2026-02-20T21:29:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:46.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:46.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:46.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:46.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:46.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_13 2026-02-20T21:29:46.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:46.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:46.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:46.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:46.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:46.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:46.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:46.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:46.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:46.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:46.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:46.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:46.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:46.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:46.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:46.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_13 2026-02-20T21:29:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:46.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:46.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:46.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:46.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:46.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:46.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:46.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:46.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_13 2026-02-20T21:29:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:46.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:46.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:46.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:46.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:46.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:46.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:46.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:46.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_13'] 2026-02-20T21:29:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:46.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_13 2026-02-20T21:29:46.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:46.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:46.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:46.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:46.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:46.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:46.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:46.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:46.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_13 2026-02-20T21:29:46.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:46.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_13 2026-02-20T21:29:46.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:46.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:46.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:46.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '' 2026-02-20T21:29:46.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:46.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:46.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:46.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:46.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:29:46.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-20T21:29:46.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-20T21:29:46.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:46.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:29:46.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:29:46.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:46.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:46.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:46.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:46.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:29:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_13 2026-02-20T21:29:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_13 '..*' 2026-02-20T21:29:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:46.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:29:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_13.status 2026-02-20T21:29:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_13.status 2026-02-20T21:29:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:46.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:46.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_13 2026-02-20T21:29:46.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:29:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=37475], [object_number=34, tag_tid=1, entry_tid=37474]]], state=connected] 2026-02-20T21:29:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.status 2026-02-20T21:29:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:29:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:29:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_13 snap 2026-02-20T21:29:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:46.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:29:46.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:29:46.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:29:47.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_13@snap 2026-02-20T21:29:47.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_13': 2026-02-20T21:29:47.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:29:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:29:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:29:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1212e7bd86a9 2026-02-20T21:29:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1212e7bd86a9 2026-02-20T21:29:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:29:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:29:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:29:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:29:47.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:21:12 2026 2026-02-20T21:29:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:21:12 2026 2026-02-20T21:29:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:22:13 2026 2026-02-20T21:29:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:29:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1212e7bd86a9 2026-02-20T21:29:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:29:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:29:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 5736794a-3cea-4cdb-b4c0-400544deb758 2026-02-20T21:29:47.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:29:47.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:47.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_13 snap 2026-02-20T21:29:47.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:47.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:29:47.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:47.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:29:47.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.export 2026-02-20T21:29:47.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_13.export 2026-02-20T21:29:47.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_13.export 2026-02-20T21:29:47.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_13@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.export 2026-02-20T21:29:47.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:29:47.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_13@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_13.export 2026-02-20T21:29:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:29:48.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_13.export 2026-02-20T21:29:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_13.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_13.export 2026-02-20T21:29:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:29:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_14 2026-02-20T21:29:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_14 snap 2026-02-20T21:29:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:48.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_14@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:29:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_14 2026-02-20T21:29:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_14 started 2026-02-20T21:29:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:29:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:29:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:29:50.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_14 started 2026-02-20T21:29:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:29:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:29:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:29:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_14 2026-02-20T21:29:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:29:50.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:29:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:29:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:29:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:29:50.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:50.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:29:50.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:50.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_14 2026-02-20T21:29:50.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:29:50.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:50.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:50.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:50.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:29:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:29:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:50.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:29:50.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:29:50.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:50.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:50.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_14 2026-02-20T21:29:50.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:50.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:50.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_14 2026-02-20T21:29:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:29:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:29:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:50.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:29:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:29:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:29:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_14 2026-02-20T21:29:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_14 2026-02-20T21:29:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:29:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:51.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:29:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:29:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:29:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:29:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:51.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:51.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:51.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:51.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_14 2026-02-20T21:29:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:29:51.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_14 2026-02-20T21:29:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:29:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:29:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:29:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:51.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:29:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:29:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:29:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:29:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:29:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:29:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:29:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:29:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:29:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:29:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:29:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:29:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:51.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_14 2026-02-20T21:29:51.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:29:51.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:29:51.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:29:51.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:29:51.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:29:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:29:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:29:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_14 2026-02-20T21:29:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:29:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:29:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:29:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:29:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_14 2026-02-20T21:29:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:29:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:29:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:29:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:29:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:29:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:29:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:29:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_14 2026-02-20T21:29:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_14 ']' 2026-02-20T21:29:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-20T21:29:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-20T21:29:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-20T21:29:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:51.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:51.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:51.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:51.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:51.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:51.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:51.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-20T21:29:51.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:51.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:51.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:51.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:51.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:51.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:51.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:51.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:51.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:51.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:51.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:51.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:51.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:51.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:51.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-20T21:29:51.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:51.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:51.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:51.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:51.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:51.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:51.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:51.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:51.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:51.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:51.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-20T21:29:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:51.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-20T21:29:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:51.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-20T21:29:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:29:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-20T21:29:51.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-20T21:29:51.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:51.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:51.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:51.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:51.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42821], [object_number=40, tag_tid=1, entry_tid=42820], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:51.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:51.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-20T21:29:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-20T21:29:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:51.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:29:51.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-20T21:29:51.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:51.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:51.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-20T21:29:51.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:51.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42821], [object_number=40, tag_tid=1, entry_tid=42820], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:51.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:51.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:51.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:51.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=42821' '!=' '' 2026-02-20T21:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=42821' = '' 2026-02-20T21:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:51.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:29:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:29:51.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:29:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:29:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:29:51.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:51.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:29:51.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:29:51.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42821 2026-02-20T21:29:51.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 42821 2026-02-20T21:29:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:29:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:29:51.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_14 2026-02-20T21:29:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_14 ']' 2026-02-20T21:29:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-20T21:29:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-20T21:29:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:51.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:51.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:51.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:51.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-20T21:29:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:51.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:51.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:51.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:51.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:51.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:51.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:51.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-20T21:29:51.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:51.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:51.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:51.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-20T21:29:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:51.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:51.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:51.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:51.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:51.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:52.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-20T21:29:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:52.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:52.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:52.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:52.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:52.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:52.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:52.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:52.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-20T21:29:52.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:52.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:52.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:52.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-20T21:29:52.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:52.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:52.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:52.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:52.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:29:52.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-20T21:29:52.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:52.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:52.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-20T21:29:52.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42821], [object_number=40, tag_tid=1, entry_tid=42820], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-20T21:29:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-20T21:29:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:29:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-20T21:29:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-20T21:29:52.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:52.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:52.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:52.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:52.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:52.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42821], [object_number=40, tag_tid=1, entry_tid=42820], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=42821' '!=' 'object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:29:52.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_14 2026-02-20T21:29:52.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:52.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_14 ']' 2026-02-20T21:29:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-20T21:29:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-20T21:29:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:52.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:52.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:52.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:52.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:52.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:52.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-20T21:29:52.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:52.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:52.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:52.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:52.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:52.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:52.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:52.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-20T21:29:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:52.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:52.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:52.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:52.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:52.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:52.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:52.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:52.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:52.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:52.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:52.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:52.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-20T21:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:52.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:52.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:52.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:52.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:52.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:52.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:52.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:52.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:52.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:52.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-20T21:29:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:52.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:52.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:52.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-20T21:29:52.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:52.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:52.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:52.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-20T21:29:52.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:52.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:52.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:52.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:52.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:29:52.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-20T21:29:52.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:52.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:52.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-20T21:29:52.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:52.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:52.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:52.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:52.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:52.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42821], [object_number=40, tag_tid=1, entry_tid=42820], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:52.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:52.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:52.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-20T21:29:52.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:52.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:52.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-20T21:29:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:29:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-20T21:29:52.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-20T21:29:52.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:52.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:52.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:52.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:52.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:52.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42821], [object_number=40, tag_tid=1, entry_tid=42820], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:52.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:52.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=42821' '!=' 'object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:52.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:29:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_14 2026-02-20T21:29:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_14 ']' 2026-02-20T21:29:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-20T21:29:53.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-20T21:29:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:53.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:53.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:53.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-20T21:29:53.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:53.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:53.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:53.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:53.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:53.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:53.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:53.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:53.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:53.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:53.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:53.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:53.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:53.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-20T21:29:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:53.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:53.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:53.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:53.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:53.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:53.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:53.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-20T21:29:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:53.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:53.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:53.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:53.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:53.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:53.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:53.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:53.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:53.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:53.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-20T21:29:53.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:53.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:53.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:53.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:53.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:53.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:53.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:53.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:53.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:53.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:53.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-20T21:29:53.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:53.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:53.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:53.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-20T21:29:53.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:53.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:53.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:53.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:53.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:29:53.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-20T21:29:53.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:53.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:53.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-20T21:29:53.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42821], [object_number=40, tag_tid=1, entry_tid=42820], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:53.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-20T21:29:53.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:53.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:53.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:53.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-20T21:29:53.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:53.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:53.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:53.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:53.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:29:53.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-20T21:29:53.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-20T21:29:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:53.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42821], [object_number=40, tag_tid=1, entry_tid=42820], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:53.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=42821' '!=' 'object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:53.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:29:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_14 2026-02-20T21:29:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_14 ']' 2026-02-20T21:29:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-20T21:29:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-20T21:29:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-20T21:29:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:55.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:55.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:55.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:55.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:55.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:55.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:55.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-20T21:29:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:55.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:55.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:55.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:55.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:55.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:55.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-20T21:29:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:55.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:55.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:55.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:55.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:55.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:55.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:55.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:55.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:55.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:55.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-20T21:29:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:55.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:55.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:55.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:55.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:55.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:55.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:55.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:55.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:55.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-20T21:29:55.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:55.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:55.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:55.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-20T21:29:55.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:29:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-20T21:29:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-20T21:29:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42821], [object_number=40, tag_tid=1, entry_tid=42820], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:55.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:55.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:55.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-20T21:29:55.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:55.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:55.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:55.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-20T21:29:55.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:55.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:55.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:55.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:55.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:29:55.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-20T21:29:55.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:55.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:55.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-20T21:29:55.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:55.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=1, entry_tid=42821], [object_number=40, tag_tid=1, entry_tid=42820], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:55.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:55.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:55.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:55.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=41, tag_tid=1, entry_tid=42821' '!=' 'object_number=41, tag_tid=1, entry_tid=42821' 2026-02-20T21:29:55.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:29:55.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:29:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_14 2026-02-20T21:29:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:29:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_14 ']' 2026-02-20T21:29:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_14' 2026-02-20T21:29:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_14 2026-02-20T21:29:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:29:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:57.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:57.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:29:57.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:29:57.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:29:57.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:29:57.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:29:57.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:29:57.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_14 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:57.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:29:57.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:29:57.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:29:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:29:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:57.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:57.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:57.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_14 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:29:57.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:29:57.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:29:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:57.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:57.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:29:57.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:58.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:58.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:58.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_14 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:58.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:29:58.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:29:58.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:58.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:58.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:29:58.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:58.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_14'] 2026-02-20T21:29:58.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:29:58.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:29:58.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:29:58.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_14 2026-02-20T21:29:58.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:29:58.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:29:58.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:29:58.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:29:58.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:29:58.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:29:58.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:29:58.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:29:58.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:29:58.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:58.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:29:58.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:58.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:29:58.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:29:58.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:29:58.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:29:58.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:29:58.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:29:58.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:58.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:58.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:29:58.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_14 2026-02-20T21:29:58.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_14 2026-02-20T21:29:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '' 2026-02-20T21:29:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:29:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:29:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-20T21:29:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-20T21:29:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:58.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:58.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:58.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:58.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_14 2026-02-20T21:29:58.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:58.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:58.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:58.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_14 '..*' 2026-02-20T21:29:58.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:29:58.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:29:58.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:29:58.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:29:58.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_14.status 2026-02-20T21:29:58.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_14.status 2026-02-20T21:29:58.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:29:58.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:58.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_14 2026-02-20T21:29:58.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:29:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:29:58.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:29:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:58.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42819], [object_number=42, tag_tid=1, entry_tid=42818]]], state=connected] 2026-02-20T21:29:58.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.status 2026-02-20T21:29:58.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:29:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_14 snap 2026-02-20T21:29:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:29:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:29:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:29:58.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:29:59.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_14@snap 2026-02-20T21:29:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_14': 2026-02-20T21:29:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:29:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:29:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:29:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 12121c69d02b 2026-02-20T21:29:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.12121c69d02b 2026-02-20T21:29:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:29:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:29:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:29:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:29:59.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:21:23 2026 2026-02-20T21:29:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:21:23 2026 2026-02-20T21:29:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:22:24 2026 2026-02-20T21:29:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:29:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 12121c69d02b 2026-02-20T21:29:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:29:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:29:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: b08e6ce1-dea1-4d1d-94cd-ae2b8b5abdbe 2026-02-20T21:29:59.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:29:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_14 snap 2026-02-20T21:29:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:29:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:29:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.export 2026-02-20T21:29:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_14.export 2026-02-20T21:29:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_14.export 2026-02-20T21:29:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_14@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.export 2026-02-20T21:29:59.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:29:59.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_14@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_14.export 2026-02-20T21:29:59.427 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:59.423+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:29:59.428 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:59.424+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:29:59.428 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:59.424+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:29:59.429 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:59.424+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:29:59.430 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:29:59.426+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:29:59.468 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:29:59.463+0000 7fc9be008640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2026-02-20T21:29:59.469 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:29:59.464+0000 7fc9c000c640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2026-02-20T21:29:59.470 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:29:59.465+0000 7fc9bf00a640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 2 2026-02-20T21:29:59.470 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:29:59.465+0000 7fc9c080d640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 2 2026-02-20T21:29:59.470 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:29:59.465+0000 7fc9c3012640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 3 2026-02-20T21:29:59.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:29:59.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_14.export 2026-02-20T21:29:59.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_14.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_14.export 2026-02-20T21:29:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:29:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:29:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_15 2026-02-20T21:29:59.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_15 snap 2026-02-20T21:29:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:29:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:29:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:29:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:29:59.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_15@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:30:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_15 2026-02-20T21:30:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_15 started 2026-02-20T21:30:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:30:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:30:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:00.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:30:01.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_15 started 2026-02-20T21:30:01.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:01.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:01.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:30:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:30:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:30:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_15 2026-02-20T21:30:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:30:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:30:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:30:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:01.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:30:01.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:30:01.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_15 2026-02-20T21:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:01.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:30:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:01.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:30:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:30:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:30:01.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:01.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:01.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_15 2026-02-20T21:30:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:01.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_15 2026-02-20T21:30:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:30:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:30:01.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:01.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_15 2026-02-20T21:30:02.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:02.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_15 2026-02-20T21:30:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:30:02.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:02.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:30:02.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:30:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:02.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:02.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:02.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:02.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:02.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:02.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:30:02.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:30:02.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_15 2026-02-20T21:30:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:02.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:02.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_15 2026-02-20T21:30:02.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:30:02.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:02.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:02.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:02.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:02.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:02.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:30:02.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:30:02.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_15 2026-02-20T21:30:02.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:30:02.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:30:02.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:30:02.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:30:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:30:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:30:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:30:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_15 2026-02-20T21:30:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:30:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:30:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:30:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:30:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_15 2026-02-20T21:30:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:30:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:30:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:30:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:30:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:30:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:30:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:02.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:30:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_15 2026-02-20T21:30:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_15 ']' 2026-02-20T21:30:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-20T21:30:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-20T21:30:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-20T21:30:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:02.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:02.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-20T21:30:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:02.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:02.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:02.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:02.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:02.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:02.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2026-02-20T21:30:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:02.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:02.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:02.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:02.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:02.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:02.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:02.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:02.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:02.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_15 2026-02-20T21:30:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:02.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:02.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:02.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:02.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:02.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:02.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:02.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:02.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-20T21:30:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-20T21:30:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:02.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:02.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:02.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:02.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:02.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19317], [object_number=16, tag_tid=1, entry_tid=19316], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-20T21:30:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-20T21:30:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:02.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:02.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:02.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:02.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:02.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:02.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19317], [object_number=16, tag_tid=1, entry_tid=19316], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:02.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:02.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19317' '!=' '' 2026-02-20T21:30:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:30:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19317' = '' 2026-02-20T21:30:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:30:02.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:02.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:30:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:30:02.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:30:02.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:30:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:02.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:30:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:30:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=19317 2026-02-20T21:30:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 19317 2026-02-20T21:30:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:30:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:30:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_15 2026-02-20T21:30:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_15 ']' 2026-02-20T21:30:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-20T21:30:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-20T21:30:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-20T21:30:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:02.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:02.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:02.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:02.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:02.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:02.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:02.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:02.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:02.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:02.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:02.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-20T21:30:02.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:02.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:02.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:02.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:02.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:02.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:02.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2026-02-20T21:30:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:02.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:02.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:02.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:02.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:03.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:03.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:03.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:03.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:03.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:03.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:03.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:03.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_15 2026-02-20T21:30:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:03.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:03.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:03.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:03.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:03.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:03.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:03.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:03.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:03.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:03.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:03.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:03.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-20T21:30:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-20T21:30:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:03.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:03.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:03.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:03.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:03.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:03.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19317], [object_number=16, tag_tid=1, entry_tid=19316], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:03.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:03.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-20T21:30:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-20T21:30:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:03.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:03.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:03.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:03.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:03.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:03.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:03.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19317], [object_number=16, tag_tid=1, entry_tid=19316], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:03.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19317' '!=' 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:30:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_15 2026-02-20T21:30:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:03.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_15 ']' 2026-02-20T21:30:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-20T21:30:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-20T21:30:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:03.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:03.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:03.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:03.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:03.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:03.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:03.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:03.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:03.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:03.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:03.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:03.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:03.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:03.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:03.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:03.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:03.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:03.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:03.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:03.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:03.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:03.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_15 2026-02-20T21:30:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:03.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:03.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:03.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:03.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:03.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-20T21:30:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-20T21:30:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:03.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:03.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:03.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:03.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:03.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:03.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:03.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19317], [object_number=16, tag_tid=1, entry_tid=19316], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:03.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:03.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-20T21:30:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-20T21:30:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:03.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:03.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:03.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:03.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19317], [object_number=16, tag_tid=1, entry_tid=19316], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19317' '!=' 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:30:04.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_15 2026-02-20T21:30:04.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:04.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:04.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:04.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:04.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_15 ']' 2026-02-20T21:30:04.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-20T21:30:04.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-20T21:30:04.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:04.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:04.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:04.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:04.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:04.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:04.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:04.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:04.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-20T21:30:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:04.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:04.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:04.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:04.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:04.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:04.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:04.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:04.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:04.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:04.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:04.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:04.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:04.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:04.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:04.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:04.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_15 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:04.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:04.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:04.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:04.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-20T21:30:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-20T21:30:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:04.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:04.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:04.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:04.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:04.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:04.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:04.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:04.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:04.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:04.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:04.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19317], [object_number=16, tag_tid=1, entry_tid=19316], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:04.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-20T21:30:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-20T21:30:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:04.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:04.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:04.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:04.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:04.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:04.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:04.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:04.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:04.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:04.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19317], [object_number=16, tag_tid=1, entry_tid=19316], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:04.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:04.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19317' '!=' 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:04.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:04.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:30:06.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_15 2026-02-20T21:30:06.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:06.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:06.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:06.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:06.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_15 ']' 2026-02-20T21:30:06.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-20T21:30:06.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-20T21:30:06.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:06.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:06.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:06.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:06.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:06.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:06.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:06.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:06.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-20T21:30:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:06.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:06.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:06.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:06.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:06.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:06.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:06.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:06.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-20T21:30:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:06.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:06.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:06.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:06.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:06.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:06.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:06.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:06.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:06.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_15 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:06.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:06.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-20T21:30:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-20T21:30:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:06.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19317], [object_number=16, tag_tid=1, entry_tid=19316], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:06.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-20T21:30:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-20T21:30:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:06.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:06.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:06.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:06.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:06.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:06.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:06.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19317], [object_number=16, tag_tid=1, entry_tid=19316], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19317' '!=' 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:30:08.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_15 2026-02-20T21:30:08.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_15 ']' 2026-02-20T21:30:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-20T21:30:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-20T21:30:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-20T21:30:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:08.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:08.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:08.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:08.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:08.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:08.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:08.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:08.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:08.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-20T21:30:08.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:08.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:08.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:08.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:08.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:08.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:09.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:09.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:09.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2026-02-20T21:30:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:09.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:09.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:09.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:09.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_15 2026-02-20T21:30:09.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:09.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:09.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:09.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:09.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-20T21:30:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:09.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-20T21:30:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:09.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:09.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19317], [object_number=16, tag_tid=1, entry_tid=19316], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:09.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:09.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:09.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-20T21:30:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-20T21:30:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:09.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:09.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19317], [object_number=16, tag_tid=1, entry_tid=19316], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19317' '!=' 'object_number=17, tag_tid=1, entry_tid=19317' 2026-02-20T21:30:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:09.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:30:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_15 2026-02-20T21:30:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_15 ']' 2026-02-20T21:30:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_15' 2026-02-20T21:30:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_15 2026-02-20T21:30:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:11.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_15 2026-02-20T21:30:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:11.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:11.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:11.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:11.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:11.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:11.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_15 2026-02-20T21:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:11.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:11.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:11.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:11.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_15 2026-02-20T21:30:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:11.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:11.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:11.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:11.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:11.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:11.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:11.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_15'] 2026-02-20T21:30:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:11.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_15 2026-02-20T21:30:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:11.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:11.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:11.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_15 2026-02-20T21:30:11.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:11.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_15 2026-02-20T21:30:11.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:11.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:11.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:11.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '' 2026-02-20T21:30:11.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:11.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:11.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:11.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:11.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:11.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:11.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:11.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:11.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:11.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:11.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:11.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_15 2026-02-20T21:30:11.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:11.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:11.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:11.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_15 '..*' 2026-02-20T21:30:11.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:11.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:11.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:30:11.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:11.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_15.status 2026-02-20T21:30:11.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_15.status 2026-02-20T21:30:11.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_15 2026-02-20T21:30:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:11.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:11.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:11.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:11.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:11.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19315], [object_number=18, tag_tid=1, entry_tid=19314]]], state=connected] 2026-02-20T21:30:11.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.status 2026-02-20T21:30:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_15 snap 2026-02-20T21:30:11.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:30:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:30:11.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:30:12.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_15@snap 2026-02-20T21:30:12.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_15': 2026-02-20T21:30:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:30:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:30:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:30:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11b317385370 2026-02-20T21:30:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11b317385370 2026-02-20T21:30:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:30:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:30:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:30:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:30:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:21:34 2026 2026-02-20T21:30:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:21:34 2026 2026-02-20T21:30:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:21:34 2026 2026-02-20T21:30:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:30:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11b317385370 2026-02-20T21:30:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:30:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:30:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 3b09137a-5453-4069-9ea4-a808cb078191 2026-02-20T21:30:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:30:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_15 snap 2026-02-20T21:30:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:30:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:30:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.export 2026-02-20T21:30:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_15.export 2026-02-20T21:30:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_15.export 2026-02-20T21:30:12.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_15@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.export 2026-02-20T21:30:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:30:12.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_15@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_15.export 2026-02-20T21:30:12.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:30:12.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_15.export 2026-02-20T21:30:12.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_15.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_15.export 2026-02-20T21:30:12.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:12.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:30:12.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_16 2026-02-20T21:30:12.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_16 snap 2026-02-20T21:30:12.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:12.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:12.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:12.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:12.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_16@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:30:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_16 2026-02-20T21:30:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:15.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_16 started 2026-02-20T21:30:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:30:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:30:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:30:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_16 started 2026-02-20T21:30:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:30:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:30:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:30:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_16 2026-02-20T21:30:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:30:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:30:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:30:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:30:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:30:16.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:16.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:30:16.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_16 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:16.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:16.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:16.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:16.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:30:16.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:30:16.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:16.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_16 2026-02-20T21:30:16.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:16.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:16.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_16 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:16.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:30:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:30:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:30:16.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:17.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_16 2026-02-20T21:30:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:17.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_16 2026-02-20T21:30:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:30:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:17.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:30:17.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:17.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:17.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:17.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:17.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_16 2026-02-20T21:30:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:17.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:17.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_16 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:17.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:30:17.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:17.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:17.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:17.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:17.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_16 2026-02-20T21:30:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:30:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:30:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:30:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:30:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_16 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_16 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_16 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_16 ']' 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:17.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:17.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:17.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:17.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:17.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:17.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:17.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:17.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-02-20T21:30:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:17.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:17.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:17.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:17.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:17.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:17.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_16 2026-02-20T21:30:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:17.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:17.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:17.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:17.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:17.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:17.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:17.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_16 2026-02-20T21:30:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:17.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:17.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:17.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:17.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-20T21:30:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-20T21:30:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:17.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-20T21:30:17.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-20T21:30:17.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:17.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:17.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-20T21:30:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:17.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:30:17.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:30:17.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:17.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=53774], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:17.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=53774], [object_number=49, tag_tid=1, entry_tid=53773], [object_number=48, tag_tid=1, entry_tid=53772], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:17.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:30:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-20T21:30:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-20T21:30:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-20T21:30:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-20T21:30:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:17.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:17.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-20T21:30:17.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:17.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:30:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:30:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=53774], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:17.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=53774], [object_number=49, tag_tid=1, entry_tid=53773], [object_number=48, tag_tid=1, entry_tid=53772], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:17.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:17.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:17.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=53774' '!=' '' 2026-02-20T21:30:17.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:30:17.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=53774' = '' 2026-02-20T21:30:17.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:17.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:30:17.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:30:17.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:17.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:30:17.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:17.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:30:17.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:17.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:30:17.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:30:17.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:30:17.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:17.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:30:17.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:17.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:30:17.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:17.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=53774 2026-02-20T21:30:17.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 53774 2026-02-20T21:30:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:30:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:17.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:30:17.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_16 2026-02-20T21:30:17.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:17.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:17.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:17.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:17.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_16 ']' 2026-02-20T21:30:17.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-20T21:30:17.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-20T21:30:17.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:17.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:17.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:17.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:17.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:17.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:17.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:17.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:17.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-20T21:30:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:17.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:17.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:17.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:17.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:17.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:17.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:17.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:17.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:17.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-02-20T21:30:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:17.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:17.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:17.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:18.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:18.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:18.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_16 2026-02-20T21:30:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:18.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:18.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:18.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:18.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:18.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:18.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_16 2026-02-20T21:30:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:18.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:18.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:18.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:18.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-20T21:30:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-20T21:30:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-20T21:30:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-20T21:30:18.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-20T21:30:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:30:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:30:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=53774], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:18.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=53774], [object_number=49, tag_tid=1, entry_tid=53773], [object_number=48, tag_tid=1, entry_tid=53772], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:18.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:18.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:30:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-20T21:30:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-20T21:30:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-20T21:30:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-20T21:30:18.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:18.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-20T21:30:18.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:18.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:30:18.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:30:18.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:18.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=53774], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:18.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=53774], [object_number=49, tag_tid=1, entry_tid=53773], [object_number=48, tag_tid=1, entry_tid=53772], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:18.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:18.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:18.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:18.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=53774' '!=' 'object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:18.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:18.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:30:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_16 2026-02-20T21:30:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:18.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_16 ']' 2026-02-20T21:30:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-20T21:30:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-20T21:30:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:18.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:18.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:18.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:18.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:18.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:18.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:18.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:18.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:18.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:18.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:18.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:18.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:18.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_16 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:18.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:18.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:18.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:18.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:18.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_16 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:18.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:18.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:18.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-20T21:30:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-20T21:30:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-20T21:30:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-20T21:30:18.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:18.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:18.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-20T21:30:18.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:30:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:30:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=53774], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:18.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=53774], [object_number=49, tag_tid=1, entry_tid=53773], [object_number=48, tag_tid=1, entry_tid=53772], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:18.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:18.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:30:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-20T21:30:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-20T21:30:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-20T21:30:18.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-20T21:30:18.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-20T21:30:18.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:30:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:30:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=53774], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:18.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=53774], [object_number=49, tag_tid=1, entry_tid=53773], [object_number=48, tag_tid=1, entry_tid=53772], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:18.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=53774' '!=' 'object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:18.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_16 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_16 ']' 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:19.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:19.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:19.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:19.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:19.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:19.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:19.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:19.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:19.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-02-20T21:30:19.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:19.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:19.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:19.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:19.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:19.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:19.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:19.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:19.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:19.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:19.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:19.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:19.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:19.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:19.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_16 2026-02-20T21:30:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:19.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:19.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:19.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:19.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:19.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:19.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:19.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:19.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:19.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:19.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_16 2026-02-20T21:30:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:19.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:19.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:19.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:19.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:19.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:19.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-20T21:30:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-20T21:30:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-20T21:30:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-20T21:30:19.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:19.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:19.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-20T21:30:19.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:30:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:30:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=53774], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=53774], [object_number=49, tag_tid=1, entry_tid=53773], [object_number=48, tag_tid=1, entry_tid=53772], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:19.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:19.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:30:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-20T21:30:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-20T21:30:19.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-20T21:30:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-20T21:30:19.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:19.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:19.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-20T21:30:19.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:30:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:30:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:19.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=53774], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=50, tag_tid=1, entry_tid=53774], [object_number=49, tag_tid=1, entry_tid=53773], [object_number=48, tag_tid=1, entry_tid=53772], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:19.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=50, tag_tid=1, entry_tid=53774' '!=' 'object_number=50, tag_tid=1, entry_tid=53774' 2026-02-20T21:30:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:19.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:30:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_16 2026-02-20T21:30:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_16 ']' 2026-02-20T21:30:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_16' 2026-02-20T21:30:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_16 2026-02-20T21:30:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:21.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:21.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:21.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:21.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:21.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:21.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:21.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_16 2026-02-20T21:30:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:21.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:21.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:21.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:21.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_16 2026-02-20T21:30:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:21.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:21.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:21.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:21.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:21.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:21.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_16 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:21.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:21.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:21.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_16'] 2026-02-20T21:30:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:21.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_16 2026-02-20T21:30:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:21.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:21.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:21.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:21.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:21.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:21.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_16 2026-02-20T21:30:21.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:21.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_16 2026-02-20T21:30:21.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:21.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:21.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:21.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '' 2026-02-20T21:30:21.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:21.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:21.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:21.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:21.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-20T21:30:21.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-20T21:30:21.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:21.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:21.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-20T21:30:21.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:30:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:30:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=53774], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:21.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=53774], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:21.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:21.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:30:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_16 2026-02-20T21:30:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:21.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_16 '..*' 2026-02-20T21:30:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:30:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_16.status 2026-02-20T21:30:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_16.status 2026-02-20T21:30:21.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:21.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:21.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_16 2026-02-20T21:30:21.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 12 2026-02-20T21:30:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 12 2026-02-20T21:30:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=53774], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:21.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=49, tag_tid=2, entry_tid=1], [object_number=48, tag_tid=2, entry_tid=0], [object_number=50, tag_tid=1, entry_tid=53774], [object_number=51, tag_tid=1, entry_tid=53771]]], state=connected] 2026-02-20T21:30:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.status 2026-02-20T21:30:21.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:30:21.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=49, tag_tid=2, entry_tid=1' -a 'object_number=49, tag_tid=2, entry_tid=1' = 'object_number=49, tag_tid=2, entry_tid=1' 2026-02-20T21:30:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_16 snap 2026-02-20T21:30:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:30:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:30:21.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:30:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_16@snap 2026-02-20T21:30:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_16': 2026-02-20T21:30:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:30:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:30:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:30:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1212951895ff 2026-02-20T21:30:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1212951895ff 2026-02-20T21:30:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:30:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:30:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:30:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:30:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:21:40 2026 2026-02-20T21:30:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:21:40 2026 2026-02-20T21:30:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:23:42 2026 2026-02-20T21:30:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:30:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1212951895ff 2026-02-20T21:30:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:30:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:30:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 5f9d4357-e233-4c2a-b9f4-e1b5971cc04e 2026-02-20T21:30:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:30:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_16 snap 2026-02-20T21:30:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:30:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:30:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.export 2026-02-20T21:30:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_16.export 2026-02-20T21:30:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_16.export 2026-02-20T21:30:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_16@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.export 2026-02-20T21:30:23.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:30:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_16@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_16.export 2026-02-20T21:30:23.117 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:30:23.113+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:30:23.119 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:30:23.114+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:30:23.119 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:30:23.114+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:30:23.119 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:30:23.115+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:30:23.119 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:30:23.115+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:30:23.119 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:30:23.115+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:30:23.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:30:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_16.export 2026-02-20T21:30:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_16.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_16.export 2026-02-20T21:30:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:30:23.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_17 2026-02-20T21:30:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_17 snap 2026-02-20T21:30:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:23.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_17@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_17 2026-02-20T21:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_17 started 2026-02-20T21:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:26.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:30:27.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_17 started 2026-02-20T21:30:27.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:27.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:27.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:27.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:30:27.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:30:27.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:30:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_17 2026-02-20T21:30:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:30:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:30:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:30:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:30:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:30:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:30:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_17 2026-02-20T21:30:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:30:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:27.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:30:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:30:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:30:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:27.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:27.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_17 2026-02-20T21:30:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-20T21:30:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_17 2026-02-20T21:30:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:30:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:28.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:30:28.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:30:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:30:28.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:30:28.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:28.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:28.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:28.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:28.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_17 2026-02-20T21:30:28.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:30:28.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:30:28.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:30:28.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:30:28.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:30:28.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:30:28.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:30:28.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:28.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_17 2026-02-20T21:30:28.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:30:28.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:28.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:30:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:30:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:30:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_17 2026-02-20T21:30:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:30:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:30:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:30:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:30:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:30:28.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:30:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:28.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:30:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_17 2026-02-20T21:30:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_17 ']' 2026-02-20T21:30:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-20T21:30:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-20T21:30:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-20T21:30:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:28.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:28.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:28.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:28.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:28.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-20T21:30:28.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-20T21:30:28.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:28.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-20T21:30:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:28.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:28.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-20T21:30:28.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-20T21:30:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-20T21:30:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-20T21:30:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-20T21:30:28.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:28.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:28.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-20T21:30:28.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:28.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:30:28.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:30:28.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:28.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898]]], state=connected] 2026-02-20T21:30:28.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896]]], state=connected] 2026-02-20T21:30:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:30:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-20T21:30:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:28.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-20T21:30:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-20T21:30:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-20T21:30:28.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-20T21:30:28.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:30:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:30:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898]]], state=connected] 2026-02-20T21:30:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896]]], state=connected] 2026-02-20T21:30:28.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=35, tag_tid=1, entry_tid=36899' '!=' '' 2026-02-20T21:30:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:30:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=35, tag_tid=1, entry_tid=36899' = '' 2026-02-20T21:30:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:30:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:30:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:30:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:30:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:28.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:30:28.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:30:28.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:30:28.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:28.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:30:28.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:28.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:30:28.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36899 2026-02-20T21:30:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 36899 2026-02-20T21:30:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:30:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:30:28.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_17 2026-02-20T21:30:28.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_17 ']' 2026-02-20T21:30:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-20T21:30:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-20T21:30:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:28.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:28.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:28.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:28.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:28.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:28.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:28.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-20T21:30:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-20T21:30:28.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-20T21:30:28.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:28.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:28.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-20T21:30:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:28.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:28.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:28.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:28.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:28.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:28.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:28.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-20T21:30:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-20T21:30:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-20T21:30:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-20T21:30:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-20T21:30:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-20T21:30:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:28.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:30:28.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:30:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898]]], state=connected] 2026-02-20T21:30:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896]]], state=connected] 2026-02-20T21:30:28.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:28.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:30:28.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-20T21:30:28.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-20T21:30:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-20T21:30:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-20T21:30:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-20T21:30:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:28.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:30:28.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:30:28.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:28.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898]]], state=connected] 2026-02-20T21:30:28.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896]]], state=connected] 2026-02-20T21:30:28.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:28.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:28.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:28.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=35, tag_tid=1, entry_tid=36899' '!=' 'object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:28.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:28.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:30:29.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_17 2026-02-20T21:30:29.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_17 ']' 2026-02-20T21:30:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-20T21:30:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-20T21:30:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-20T21:30:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:29.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:29.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:29.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-20T21:30:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-20T21:30:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:29.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-20T21:30:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:29.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:29.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:29.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:29.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-20T21:30:29.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-20T21:30:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-20T21:30:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-20T21:30:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-20T21:30:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-20T21:30:29.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:30:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:30:29.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898]]], state=connected] 2026-02-20T21:30:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896]]], state=connected] 2026-02-20T21:30:29.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:29.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:30:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-20T21:30:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-20T21:30:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-20T21:30:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-20T21:30:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-20T21:30:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:30:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:30:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898]]], state=connected] 2026-02-20T21:30:29.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896]]], state=connected] 2026-02-20T21:30:29.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:29.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:29.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=35, tag_tid=1, entry_tid=36899' '!=' 'object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:29.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:30:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_17 2026-02-20T21:30:30.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_17 ']' 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:30.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:30.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:30.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:30.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-20T21:30:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-20T21:30:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:30.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:30.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:30.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-20T21:30:30.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-20T21:30:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:30.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-20T21:30:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-20T21:30:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-20T21:30:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-20T21:30:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:30.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:30:30.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:30:30.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:30.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898]]], state=connected] 2026-02-20T21:30:30.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896]]], state=connected] 2026-02-20T21:30:30.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:30.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:30:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-20T21:30:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-20T21:30:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:30.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:30.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:30.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:30.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-20T21:30:30.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-20T21:30:30.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:30.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:30.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-20T21:30:30.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:30.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:30:30.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:30:30.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:30.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898]]], state=connected] 2026-02-20T21:30:30.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898], [object_number=33, tag_tid=1, entry_tid=36897], [object_number=32, tag_tid=1, entry_tid=36896]]], state=connected] 2026-02-20T21:30:30.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:30.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:30.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:30.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=35, tag_tid=1, entry_tid=36899' '!=' 'object_number=35, tag_tid=1, entry_tid=36899' 2026-02-20T21:30:30.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:30.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:30:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_17 2026-02-20T21:30:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:32.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_17 ']' 2026-02-20T21:30:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_17' 2026-02-20T21:30:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_17 2026-02-20T21:30:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:32.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:32.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_17 2026-02-20T21:30:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:32.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:32.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:32.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:32.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_17 2026-02-20T21:30:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_17'] 2026-02-20T21:30:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:32.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:32.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_17 2026-02-20T21:30:32.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:32.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:32.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:32.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:32.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_17 2026-02-20T21:30:32.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_17 2026-02-20T21:30:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '' 2026-02-20T21:30:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-20T21:30:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-20T21:30:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:32.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-20T21:30:32.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:30:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:30:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898]]], state=connected] 2026-02-20T21:30:32.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898]]], state=connected] 2026-02-20T21:30:32.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:32.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:30:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_17 2026-02-20T21:30:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_17 '..*' 2026-02-20T21:30:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:32.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:30:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_17.status 2026-02-20T21:30:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_17.status 2026-02-20T21:30:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_17 2026-02-20T21:30:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:32.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:30:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:30:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898]]], state=connected] 2026-02-20T21:30:32.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36899], [object_number=34, tag_tid=1, entry_tid=36898]]], state=connected] 2026-02-20T21:30:32.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.status 2026-02-20T21:30:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:30:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:30:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_17 snap 2026-02-20T21:30:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:30:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:30:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:30:33.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_17@snap 2026-02-20T21:30:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_17': 2026-02-20T21:30:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:30:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:30:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:30:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ec6a52a713 2026-02-20T21:30:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ec6a52a713 2026-02-20T21:30:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:30:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:30:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:30:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:30:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:21:53 2026 2026-02-20T21:30:33.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:21:53 2026 2026-02-20T21:30:33.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:24:10 2026 2026-02-20T21:30:33.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:30:33.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ec6a52a713 2026-02-20T21:30:33.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:30:33.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:30:33.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 1fa2e7cc-8ee4-43e1-9607-216a31c3e96d 2026-02-20T21:30:33.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:30:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_17 snap 2026-02-20T21:30:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:30:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:30:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.export 2026-02-20T21:30:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_17.export 2026-02-20T21:30:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_17.export 2026-02-20T21:30:33.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_17@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.export 2026-02-20T21:30:33.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:30:33.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_17@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_17.export 2026-02-20T21:30:33.447 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:30:33.442+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:30:33.447 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:30:33.442+0000 7f412a4c7640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:30:33.447 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:30:33.442+0000 7f4128cc4640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:30:33.448 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:30:33.443+0000 7f412cccc640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:30:33.448 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:30:33.443+0000 7f412bcca640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:30:33.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:30:33.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_17.export 2026-02-20T21:30:33.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_17.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_17.export 2026-02-20T21:30:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:30:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_18 2026-02-20T21:30:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_18 snap 2026-02-20T21:30:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:33.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:33.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:33.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:33.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_18@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:30:34.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_18 2026-02-20T21:30:34.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:34.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:34.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:34.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_18 started 2026-02-20T21:30:34.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:34.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:34.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:34.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:30:34.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:30:34.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:34.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:30:35.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_18 started 2026-02-20T21:30:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:30:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:30:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:30:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_18 2026-02-20T21:30:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:30:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:30:35.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:30:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:30:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:30:35.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:35.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:30:35.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_18 2026-02-20T21:30:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:30:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:30:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:30:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:30:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:30:35.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:35.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_18 2026-02-20T21:30:36.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:36.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:36.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_18 2026-02-20T21:30:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:30:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:36.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:30:36.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:36.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:36.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:36.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:36.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_18 2026-02-20T21:30:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:36.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_18 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:30:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:36.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_18 2026-02-20T21:30:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:30:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:30:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:30:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:30:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_18 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_18 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:36.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:30:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-20T21:30:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-20T21:30:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-20T21:30:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-20T21:30:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:36.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:36.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:36.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:36.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:36.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:36.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:36.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:36.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:36.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:36.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:36.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-20T21:30:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:36.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:36.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:36.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:36.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:36.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-20T21:30:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:36.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:36.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:36.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:36.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:36.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-20T21:30:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:36.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-20T21:30:36.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:36.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:36.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:36.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:36.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:30:36.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-20T21:30:36.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:36.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:36.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-20T21:30:36.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:36.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:36.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:36.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:36.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:36.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17806], [object_number=17, tag_tid=1, entry_tid=17805], [object_number=16, tag_tid=1, entry_tid=17804], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:36.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:36.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-20T21:30:36.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-20T21:30:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:30:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-20T21:30:36.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-20T21:30:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:36.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:36.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17806], [object_number=17, tag_tid=1, entry_tid=17805], [object_number=16, tag_tid=1, entry_tid=17804], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:36.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17806' '!=' '' 2026-02-20T21:30:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:30:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17806' = '' 2026-02-20T21:30:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:30:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:36.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:30:36.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:36.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:30:36.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:36.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:30:36.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:36.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:30:36.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:36.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:30:36.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:36.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:30:36.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:36.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=17806 2026-02-20T21:30:36.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 17806 2026-02-20T21:30:36.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:30:36.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:36.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:30:36.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-20T21:30:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-20T21:30:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-20T21:30:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-20T21:30:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:36.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:36.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:36.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:36.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:36.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:36.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-20T21:30:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:36.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:36.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:36.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:36.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:36.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:36.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-20T21:30:36.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:36.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:36.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:36.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:36.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:36.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:36.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:36.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-20T21:30:36.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:36.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:36.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:36.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:36.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:36.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:36.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:36.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:36.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:36.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:36.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:36.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:36.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:36.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:36.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:36.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:36.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:36.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:36.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:36.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:36.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:36.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:36.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:36.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:36.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-20T21:30:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-20T21:30:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:30:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-20T21:30:36.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-20T21:30:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:36.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17806], [object_number=17, tag_tid=1, entry_tid=17805], [object_number=16, tag_tid=1, entry_tid=17804], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:36.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:36.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-20T21:30:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:37.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-20T21:30:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:30:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-20T21:30:37.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-20T21:30:37.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:37.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:37.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:37.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17806], [object_number=17, tag_tid=1, entry_tid=17805], [object_number=16, tag_tid=1, entry_tid=17804], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:37.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:37.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17806' '!=' 'object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:37.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:37.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:30:37.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-20T21:30:37.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:37.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:37.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:37.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:37.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-20T21:30:37.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-20T21:30:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-20T21:30:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:37.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-20T21:30:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:37.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:37.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:37.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:37.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-20T21:30:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-20T21:30:37.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:37.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:37.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:37.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:37.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:37.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:37.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:37.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:37.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:37.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-20T21:30:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-20T21:30:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:30:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-20T21:30:37.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-20T21:30:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:37.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17806], [object_number=17, tag_tid=1, entry_tid=17805], [object_number=16, tag_tid=1, entry_tid=17804], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:37.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:37.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:37.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-20T21:30:37.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:37.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:37.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:37.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-20T21:30:37.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:37.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:30:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-20T21:30:37.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-20T21:30:37.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:37.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17806], [object_number=17, tag_tid=1, entry_tid=17805], [object_number=16, tag_tid=1, entry_tid=17804], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:37.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17806' '!=' 'object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:37.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:30:38.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-20T21:30:38.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:38.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:38.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-20T21:30:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-20T21:30:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-20T21:30:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:38.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:38.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:38.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:38.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:38.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:38.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:38.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:38.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:38.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:38.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:38.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:38.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:38.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:38.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:38.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-20T21:30:38.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:38.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:38.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:38.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:38.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:38.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:38.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:38.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:38.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:38.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:38.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-20T21:30:38.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:38.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:38.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:38.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:38.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:38.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:38.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:38.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:38.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:38.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:38.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:38.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:38.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:38.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:38.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:38.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:38.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:38.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:38.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:38.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:38.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:38.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:38.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-20T21:30:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-20T21:30:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:30:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-20T21:30:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-20T21:30:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17806], [object_number=17, tag_tid=1, entry_tid=17805], [object_number=16, tag_tid=1, entry_tid=17804], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:38.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-20T21:30:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:38.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-20T21:30:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:30:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-20T21:30:38.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-20T21:30:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:38.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:38.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:38.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:38.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:38.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17806], [object_number=17, tag_tid=1, entry_tid=17805], [object_number=16, tag_tid=1, entry_tid=17804], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:38.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:38.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:38.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:38.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17806' '!=' 'object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:38.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:38.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:30:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-20T21:30:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-20T21:30:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-20T21:30:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-20T21:30:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:40.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:40.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:40.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:40.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:40.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:40.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-20T21:30:40.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:40.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:40.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:40.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:40.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:40.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:40.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:40.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:40.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:40.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:40.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:40.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:40.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:40.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-20T21:30:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:40.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-20T21:30:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:40.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:40.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:40.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:40.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-20T21:30:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-20T21:30:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:30:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-20T21:30:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-20T21:30:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:40.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:40.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17806], [object_number=17, tag_tid=1, entry_tid=17805], [object_number=16, tag_tid=1, entry_tid=17804], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:40.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:40.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:40.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-20T21:30:40.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:40.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:40.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-20T21:30:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:30:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-20T21:30:40.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-20T21:30:40.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:40.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:40.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:40.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:40.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17806], [object_number=17, tag_tid=1, entry_tid=17805], [object_number=16, tag_tid=1, entry_tid=17804], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17806' '!=' 'object_number=18, tag_tid=1, entry_tid=17806' 2026-02-20T21:30:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:30:42.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_18 2026-02-20T21:30:42.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:42.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:42.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:42.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:42.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_18 ']' 2026-02-20T21:30:42.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_18' 2026-02-20T21:30:42.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_18 2026-02-20T21:30:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:42.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_18 2026-02-20T21:30:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:42.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:42.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:42.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:42.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:42.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:42.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:42.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_18 2026-02-20T21:30:42.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:42.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:42.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:42.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:42.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:42.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:42.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_18'] 2026-02-20T21:30:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_18 2026-02-20T21:30:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:42.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:42.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:42.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:42.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:42.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:42.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_18 2026-02-20T21:30:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_18 2026-02-20T21:30:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:42.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '' 2026-02-20T21:30:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:30:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-20T21:30:42.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-20T21:30:42.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:42.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:42.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:42.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_18 2026-02-20T21:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_18 '..*' 2026-02-20T21:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_18.status 2026-02-20T21:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_18.status 2026-02-20T21:30:42.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:42.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:42.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_18 2026-02-20T21:30:42.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:42.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:30:42.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:30:42.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:42.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:42.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17806], [object_number=19, tag_tid=1, entry_tid=17803]]], state=connected] 2026-02-20T21:30:42.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.status 2026-02-20T21:30:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:30:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_18 snap 2026-02-20T21:30:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:30:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:30:42.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:30:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_18@snap 2026-02-20T21:30:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_18': 2026-02-20T21:30:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:30:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:30:43.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:30:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ff7ecdaa4d 2026-02-20T21:30:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ff7ecdaa4d 2026-02-20T21:30:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:30:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:30:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:30:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:30:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:22:03 2026 2026-02-20T21:30:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:22:03 2026 2026-02-20T21:30:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:22:03 2026 2026-02-20T21:30:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:30:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ff7ecdaa4d 2026-02-20T21:30:43.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:30:43.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:30:43.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 1816147c-02b1-44fd-9adf-c552b013ef88 2026-02-20T21:30:43.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:30:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_18 snap 2026-02-20T21:30:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:30:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:30:43.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.export 2026-02-20T21:30:43.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_18.export 2026-02-20T21:30:43.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_18.export 2026-02-20T21:30:43.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_18@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.export 2026-02-20T21:30:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:30:43.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_18@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_18.export 2026-02-20T21:30:44.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:30:44.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_18.export 2026-02-20T21:30:44.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_18.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_18.export 2026-02-20T21:30:44.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:44.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:30:44.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_19 2026-02-20T21:30:44.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_19 snap 2026-02-20T21:30:44.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:44.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:44.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:44.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:44.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_19@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:30:47.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_19 2026-02-20T21:30:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_19 started 2026-02-20T21:30:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:30:47.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:30:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:30:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_19 started 2026-02-20T21:30:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:30:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:30:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:30:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_19 2026-02-20T21:30:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:30:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:30:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:30:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:30:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:30:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:30:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_19 2026-02-20T21:30:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:30:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:30:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:30:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:30:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:30:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:48.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:48.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:48.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:48.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_19 2026-02-20T21:30:48.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:48.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:48.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_19 2026-02-20T21:30:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:30:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:30:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:30:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:48.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:48.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:30:48.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:30:48.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:48.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:48.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_19 2026-02-20T21:30:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:48.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:48.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_19 2026-02-20T21:30:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:30:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:48.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:30:48.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:30:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:30:48.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:48.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_19 2026-02-20T21:30:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:48.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_19 2026-02-20T21:30:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:30:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:48.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:30:48.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:30:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:30:48.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:30:48.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:48.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:48.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:48.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:48.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_19 2026-02-20T21:30:48.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:30:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:30:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:30:48.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:30:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:30:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:30:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:30:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_19 2026-02-20T21:30:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:30:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:30:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:30:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:30:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_19 2026-02-20T21:30:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:30:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:30:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:30:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:30:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:30:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:30:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:30:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_19 2026-02-20T21:30:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_19 ']' 2026-02-20T21:30:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-20T21:30:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-20T21:30:48.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:48.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:48.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:48.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:48.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:48.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:48.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:48.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:48.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-20T21:30:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:48.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:48.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:48.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:48.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:48.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:48.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:48.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:48.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-20T21:30:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:48.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:48.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:48.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:48.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:48.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:48.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:48.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:48.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:48.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:48.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:48.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:48.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-20T21:30:48.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:48.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:48.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:48.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:48.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:48.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:48.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:48.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:48.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:48.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:48.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:48.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:48.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_19 2026-02-20T21:30:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:48.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:48.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:48.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:48.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:48.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:48.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:48.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:48.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:48.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:48.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:48.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:48.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:48.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:48.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-20T21:30:48.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:48.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:48.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:48.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-20T21:30:48.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:48.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:48.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:48.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:48.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-20T21:30:48.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-20T21:30:48.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:48.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:48.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-20T21:30:48.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:48.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:30:48.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:30:48.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:48.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490]]], state=connected] 2026-02-20T21:30:48.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=27492], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490], [object_number=25, tag_tid=1, entry_tid=27489]]], state=connected] 2026-02-20T21:30:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:30:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-20T21:30:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-20T21:30:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:48.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:48.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-20T21:30:48.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-20T21:30:48.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:48.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:48.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-20T21:30:48.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:30:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:30:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490]]], state=connected] 2026-02-20T21:30:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=27492], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490], [object_number=25, tag_tid=1, entry_tid=27489]]], state=connected] 2026-02-20T21:30:48.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=27492' '!=' '' 2026-02-20T21:30:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:30:48.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=27492' = '' 2026-02-20T21:30:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:30:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:30:48.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:30:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:30:48.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:30:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:30:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:30:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:30:48.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:48.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:30:48.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27492 2026-02-20T21:30:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 27492 2026-02-20T21:30:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:30:48.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:30:48.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_19 2026-02-20T21:30:48.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:48.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:48.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:48.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:48.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_19 ']' 2026-02-20T21:30:48.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-20T21:30:48.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-20T21:30:48.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:48.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:48.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:48.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-20T21:30:48.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:48.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:48.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:48.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:48.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:48.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:48.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:48.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:48.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:48.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:48.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:48.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:48.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:48.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:48.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:48.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:48.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:49.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-20T21:30:49.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:49.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:49.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:49.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:49.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:49.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:49.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:49.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-20T21:30:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:49.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:49.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:49.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:49.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_19 2026-02-20T21:30:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:49.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:49.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:49.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:49.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-20T21:30:49.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:49.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:49.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:49.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-20T21:30:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-20T21:30:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-20T21:30:49.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-20T21:30:49.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:30:49.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:30:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490]]], state=connected] 2026-02-20T21:30:49.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=27492], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490], [object_number=25, tag_tid=1, entry_tid=27489]]], state=connected] 2026-02-20T21:30:49.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:49.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:30:49.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-20T21:30:49.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-20T21:30:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-20T21:30:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-20T21:30:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-20T21:30:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:30:49.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:30:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490]]], state=connected] 2026-02-20T21:30:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=27492], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490], [object_number=25, tag_tid=1, entry_tid=27489]]], state=connected] 2026-02-20T21:30:49.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=27492' '!=' 'object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:30:49.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_19 2026-02-20T21:30:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_19 ']' 2026-02-20T21:30:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-20T21:30:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-20T21:30:49.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:49.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:49.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:49.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-20T21:30:49.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:49.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:49.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:49.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:49.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:49.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:49.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:49.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:49.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:49.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:49.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:49.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:49.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:49.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:49.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:49.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-20T21:30:49.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:49.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:49.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:49.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:49.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:49.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:49.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:49.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:49.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:49.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:49.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:49.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:49.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:49.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-20T21:30:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:49.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:49.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:49.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:49.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:49.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:49.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:49.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:49.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:49.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:49.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_19 2026-02-20T21:30:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:49.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:49.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:49.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:49.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-20T21:30:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-20T21:30:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:49.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-20T21:30:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-20T21:30:49.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:49.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:49.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-20T21:30:49.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:49.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:30:49.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:30:49.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:49.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490]]], state=connected] 2026-02-20T21:30:49.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=27492], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490], [object_number=25, tag_tid=1, entry_tid=27489]]], state=connected] 2026-02-20T21:30:49.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:49.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:30:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-20T21:30:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-20T21:30:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:49.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-20T21:30:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-20T21:30:49.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-20T21:30:49.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:30:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:30:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490]]], state=connected] 2026-02-20T21:30:49.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=27492], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490], [object_number=25, tag_tid=1, entry_tid=27489]]], state=connected] 2026-02-20T21:30:49.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:49.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:49.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:49.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=27492' '!=' 'object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:49.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:49.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:30:50.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_19 2026-02-20T21:30:50.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:50.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_19 ']' 2026-02-20T21:30:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-20T21:30:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-20T21:30:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:50.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:50.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:50.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:50.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-20T21:30:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:50.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:50.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:50.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:50.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:50.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:50.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:50.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:50.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:50.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:50.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:50.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:50.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:50.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:50.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:50.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:50.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:50.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:50.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:50.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-20T21:30:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:50.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:50.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:50.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:50.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:50.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:50.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:50.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:50.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:50.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:50.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:50.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:50.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-20T21:30:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:50.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:50.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:50.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:50.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:50.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:50.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:50.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:50.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_19 2026-02-20T21:30:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:50.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:50.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:50.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:50.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:50.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:50.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:50.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:50.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:50.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:50.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:50.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:50.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-20T21:30:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-20T21:30:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-20T21:30:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-20T21:30:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-20T21:30:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:30:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:30:51.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:51.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490]]], state=connected] 2026-02-20T21:30:51.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=27492], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490], [object_number=25, tag_tid=1, entry_tid=27489]]], state=connected] 2026-02-20T21:30:51.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:51.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:30:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-20T21:30:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:51.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-20T21:30:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-20T21:30:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-20T21:30:51.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-20T21:30:51.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:30:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:30:51.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:51.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490]]], state=connected] 2026-02-20T21:30:51.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=27492], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490], [object_number=25, tag_tid=1, entry_tid=27489]]], state=connected] 2026-02-20T21:30:51.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:51.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:51.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:51.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=27492' '!=' 'object_number=24, tag_tid=1, entry_tid=27492' 2026-02-20T21:30:51.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:51.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:30:52.252 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7889890794662111) > weight (0.25) 2026-02-20T21:30:52.253 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.3263667061261385) > weight (0.25) 2026-02-20T21:30:52.253 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5319880686680305) > weight (0.25) 2026-02-20T21:30:52.253 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-20T21:30:52.253 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-20T21:30:52.253 INFO:tasks.rbd_mirror_thrash:waiting for 2.607170678134658 secs before reviving daemons 2026-02-20T21:30:52.254 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:30:52.252+0000 7f3e44873640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 47374) UID: 1000 2026-02-20T21:30:52.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_19 2026-02-20T21:30:52.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:52.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_19 ']' 2026-02-20T21:30:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_19' 2026-02-20T21:30:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_19 2026-02-20T21:30:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:52.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_19 2026-02-20T21:30:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:52.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:52.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:52.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:52.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:52.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:52.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:52.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:52.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:52.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:52.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:52.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_19 2026-02-20T21:30:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:52.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:52.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:52.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:52.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:52.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:52.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:52.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:52.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_19 2026-02-20T21:30:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:52.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:52.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:52.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:52.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:52.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_19'] 2026-02-20T21:30:52.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:52.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_19 2026-02-20T21:30:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:52.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:52.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:52.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:52.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:52.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:52.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:52.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:52.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:52.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:52.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_19 2026-02-20T21:30:52.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:52.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_19 2026-02-20T21:30:52.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:52.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:52.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:52.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '' 2026-02-20T21:30:52.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:52.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:52.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-20T21:30:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-20T21:30:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:52.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-20T21:30:52.944 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:52.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:30:52.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:30:52.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:52.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490]]], state=connected] 2026-02-20T21:30:52.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490]]], state=connected] 2026-02-20T21:30:52.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:30:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_19 2026-02-20T21:30:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_19 '..*' 2026-02-20T21:30:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:30:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_19.status 2026-02-20T21:30:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_19.status 2026-02-20T21:30:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:52.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:52.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_19 2026-02-20T21:30:52.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:53.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:30:53.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:30:53.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:53.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490]]], state=connected] 2026-02-20T21:30:53.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27491], [object_number=26, tag_tid=1, entry_tid=27490]]], state=connected] 2026-02-20T21:30:53.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.status 2026-02-20T21:30:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:30:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:30:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_19 snap 2026-02-20T21:30:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:30:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:30:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:30:54.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_19@snap 2026-02-20T21:30:54.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_19': 2026-02-20T21:30:54.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:30:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:30:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:30:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11b335b60447 2026-02-20T21:30:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11b335b60447 2026-02-20T21:30:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:30:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:30:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:30:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:30:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:22:08 2026 2026-02-20T21:30:54.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:22:08 2026 2026-02-20T21:30:54.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:23:09 2026 2026-02-20T21:30:54.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:30:54.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11b335b60447 2026-02-20T21:30:54.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:30:54.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:30:54.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: e32f86ab-f642-48b5-991a-bf1357b7c7f9 2026-02-20T21:30:54.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:30:54.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_19 snap 2026-02-20T21:30:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:30:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:30:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.export 2026-02-20T21:30:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_19.export 2026-02-20T21:30:54.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_19.export 2026-02-20T21:30:54.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_19@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.export 2026-02-20T21:30:54.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:30:54.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_19@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_19.export 2026-02-20T21:30:54.208 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:30:54.204+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:30:54.208 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:30:54.204+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:30:54.209 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:30:54.204+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:30:54.210 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:30:54.204+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:30:54.210 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:30:54.204+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:30:54.241 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:30:54.237+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:30:54.241 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:30:54.237+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:30:54.242 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:30:54.237+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:30:54.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:30:54.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_19.export 2026-02-20T21:30:54.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_19.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_19.export 2026-02-20T21:30:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:30:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_20 2026-02-20T21:30:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_20 snap 2026-02-20T21:30:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:30:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:30:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_20@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:30:54.861 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-20T21:30:54.861 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:30:54.862 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-20T21:30:54.862 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-20T21:30:54.862 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-20T21:30:54.866 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-20T21:30:54.866 INFO:tasks.rbd_mirror_thrash:waiting for 110.72800453899342 secs before thrashing 2026-02-20T21:30:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_20 2026-02-20T21:30:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:56.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:30:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_20 started 2026-02-20T21:30:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:30:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:30:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:30:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:56.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:30:57.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_20 started 2026-02-20T21:30:57.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:57.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:57.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:30:57.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:30:57.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:30:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:30:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_20 2026-02-20T21:30:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:30:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:30:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:30:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:30:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:30:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:30:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:57.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_20 2026-02-20T21:30:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:30:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:57.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:57.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:57.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:30:57.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:30:57.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:57.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_20 2026-02-20T21:30:57.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:30:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:57.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_20 2026-02-20T21:30:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:30:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:57.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:57.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:30:57.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:30:57.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:57.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:57.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:57.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:57.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_20 2026-02-20T21:30:57.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:30:57.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:57.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:57.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:57.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_20 2026-02-20T21:30:57.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:30:57.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:57.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:57.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:57.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:30:57.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:30:57.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:57.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:57.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_20 2026-02-20T21:30:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:30:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:30:57.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_20 2026-02-20T21:30:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:30:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:30:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:30:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:57.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:30:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:30:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:30:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:30:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:30:57.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:30:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:30:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:30:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:30:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:30:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:30:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:30:57.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:57.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:57.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:57.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:57.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_20 2026-02-20T21:30:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:30:57.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:30:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:30:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:30:57.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:30:57.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:30:57.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:30:57.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:57.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_20 2026-02-20T21:30:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:30:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:30:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:30:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:30:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:30:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_20 2026-02-20T21:30:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:30:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:30:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:30:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:30:57.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:30:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:30:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:30:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:30:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:57.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:30:57.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_20 2026-02-20T21:30:57.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:57.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:30:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_20 ']' 2026-02-20T21:30:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-20T21:30:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-20T21:30:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:57.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:57.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:57.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:57.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:57.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:57.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:57.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-20T21:30:57.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:57.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:57.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:57.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:57.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:57.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:57.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:57.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:57.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:57.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:57.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:57.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:57.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:57.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:57.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:57.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:57.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:57.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:57.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:57.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:57.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:57.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:57.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:57.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:57.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:57.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:57.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:57.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:57.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:57.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:57.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:57.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:57.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:57.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-20T21:30:57.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:30:57.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:57.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:57.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:57.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2026-02-20T21:30:57.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:57.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:57.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:57.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:57.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:57.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:57.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:57.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:57.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:57.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:57.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:57.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:57.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:57.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2026-02-20T21:30:57.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:30:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:57.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:57.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:57.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_20 2026-02-20T21:30:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:57.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:57.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:57.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:57.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:57.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:57.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:57.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_20 2026-02-20T21:30:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:30:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_20 2026-02-20T21:30:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:57.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:57.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:57.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:57.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_20 2026-02-20T21:30:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:57.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-20T21:30:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:30:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-20T21:30:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:30:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-20T21:30:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-20T21:30:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-20T21:30:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:30:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:30:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39122], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:30:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39122], [object_number=37, tag_tid=1, entry_tid=39121], [object_number=36, tag_tid=1, entry_tid=39120], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:30:57.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:30:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-20T21:30:57.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:30:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-20T21:30:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:30:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-20T21:30:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-20T21:30:57.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-20T21:30:57.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:30:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:30:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39122], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:30:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39122], [object_number=37, tag_tid=1, entry_tid=39121], [object_number=36, tag_tid=1, entry_tid=39120], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:30:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:30:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:30:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=39122' '!=' '' 2026-02-20T21:30:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:30:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=39122' = '' 2026-02-20T21:30:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:30:57.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:30:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:30:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:30:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:30:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:30:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:57.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:30:57.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:30:57.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:30:57.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:57.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:30:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:30:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:30:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:30:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=39122 2026-02-20T21:30:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 39122 2026-02-20T21:30:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:30:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:57.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:30:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_20 2026-02-20T21:30:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:58.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:30:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_20 ']' 2026-02-20T21:30:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-20T21:30:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-20T21:30:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:58.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:58.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:58.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:58.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:58.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:58.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-20T21:30:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:58.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:58.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:58.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:58.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-20T21:30:58.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:30:58.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:58.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2026-02-20T21:30:58.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:58.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:58.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:58.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:58.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:58.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2026-02-20T21:30:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:30:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:58.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:58.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_20 2026-02-20T21:30:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:58.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:58.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:58.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:58.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:58.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_20 2026-02-20T21:30:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:30:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:58.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:58.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_20 2026-02-20T21:30:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:58.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:58.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:58.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:58.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:58.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:58.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:58.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:58.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_20 2026-02-20T21:30:58.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:58.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-20T21:30:58.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:58.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:58.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:30:58.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-20T21:30:58.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:58.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:58.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:30:58.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:58.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-20T21:30:58.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-20T21:30:58.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:58.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:58.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-20T21:30:58.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:58.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:30:58.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:30:58.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:58.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39122], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:30:58.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39122], [object_number=37, tag_tid=1, entry_tid=39121], [object_number=36, tag_tid=1, entry_tid=39120], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:30:58.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:30:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-20T21:30:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:58.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:58.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:30:58.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-20T21:30:58.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:58.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:58.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:30:58.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:58.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-20T21:30:58.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-20T21:30:58.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-20T21:30:58.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:30:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:30:58.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39122], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:30:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39122], [object_number=37, tag_tid=1, entry_tid=39121], [object_number=36, tag_tid=1, entry_tid=39120], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:30:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:30:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:30:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=39122' '!=' 'object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:30:58.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:58.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:30:58.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_20 2026-02-20T21:30:58.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:58.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:58.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:30:58.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:58.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_20 ']' 2026-02-20T21:30:58.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-20T21:30:58.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-20T21:30:58.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:58.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:58.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:58.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:58.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:58.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:58.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:58.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:58.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:58.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:58.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:58.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-20T21:30:58.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:30:58.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:58.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:58.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:30:58.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:30:58.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:58.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:58.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:30:58.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2026-02-20T21:30:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:30:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:59.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_20 2026-02-20T21:30:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:30:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:30:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:30:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:30:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:59.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:59.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:30:59.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:30:59.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:59.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:59.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:30:59.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:30:59.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:30:59.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:59.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:59.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:30:59.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_20 2026-02-20T21:30:59.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:30:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:30:59.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:30:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_20 2026-02-20T21:30:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:30:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:30:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:30:59.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:30:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:30:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:30:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:30:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:59.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:30:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:30:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:30:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:30:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:30:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:30:59.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:30:59.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:59.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:59.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:30:59.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_20 2026-02-20T21:30:59.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:30:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-20T21:30:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:30:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-20T21:30:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:59.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:30:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:30:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-20T21:30:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-20T21:30:59.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-20T21:30:59.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:30:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:30:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39122], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:30:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39122], [object_number=37, tag_tid=1, entry_tid=39121], [object_number=36, tag_tid=1, entry_tid=39120], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:30:59.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:59.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:30:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-20T21:30:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:30:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-20T21:30:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:30:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:30:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:30:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:30:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-20T21:30:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-20T21:30:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:30:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-20T21:30:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:59.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:30:59.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:30:59.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:30:59.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39122], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:30:59.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39122], [object_number=37, tag_tid=1, entry_tid=39121], [object_number=36, tag_tid=1, entry_tid=39120], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:30:59.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:30:59.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:30:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:30:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=39122' '!=' 'object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:30:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:30:59.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:30:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_20 2026-02-20T21:30:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:30:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:30:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:30:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_20 ']' 2026-02-20T21:30:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-20T21:30:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-20T21:30:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:30:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:30:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:30:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:30:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:30:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:30:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:30:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:30:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:30:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-20T21:30:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:30:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:30:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:30:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:30:59.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:30:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:30:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:30:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:30:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:30:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:30:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:30:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:30:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:30:59.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:30:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:30:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:30:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:30:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:30:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:30:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:30:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:30:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:30:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:30:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:30:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:30:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:30:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:30:59.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:59.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:30:59.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-20T21:31:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:31:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2026-02-20T21:31:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:31:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:31:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:31:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:31:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:31:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:31:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:31:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:31:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:31:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2026-02-20T21:31:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:31:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_20 2026-02-20T21:31:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:31:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:31:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:31:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:31:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:31:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:31:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:31:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:31:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:31:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:00.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:00.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:00.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:00.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:00.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:31:00.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:31:00.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:31:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:00.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:00.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:00.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_20 2026-02-20T21:31:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:31:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:00.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:00.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_20 2026-02-20T21:31:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:31:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:31:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:31:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:31:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:31:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:31:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:31:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:31:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:31:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:00.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:31:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:31:00.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:31:00.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:00.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:00.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:00.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_20 2026-02-20T21:31:00.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:00.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-20T21:31:00.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:00.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:00.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:31:00.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-20T21:31:00.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:31:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-20T21:31:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-20T21:31:00.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:00.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:31:00.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-20T21:31:00.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:31:00.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:31:00.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:31:00.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:00.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39122], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:31:00.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39122], [object_number=37, tag_tid=1, entry_tid=39121], [object_number=36, tag_tid=1, entry_tid=39120], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:31:00.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:31:00.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:31:00.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-20T21:31:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:31:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-20T21:31:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:31:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-20T21:31:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-20T21:31:00.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:00.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:31:00.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-20T21:31:00.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:31:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:31:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:31:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39122], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:31:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=38, tag_tid=1, entry_tid=39122], [object_number=37, tag_tid=1, entry_tid=39121], [object_number=36, tag_tid=1, entry_tid=39120], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:31:00.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:31:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:31:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:31:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=38, tag_tid=1, entry_tid=39122' '!=' 'object_number=38, tag_tid=1, entry_tid=39122' 2026-02-20T21:31:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:00.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:31:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_20 2026-02-20T21:31:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:31:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_20 ']' 2026-02-20T21:31:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_20' 2026-02-20T21:31:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_20 2026-02-20T21:31:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_20 2026-02-20T21:31:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:01.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:01.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:01.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:01.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_20 2026-02-20T21:31:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:31:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:01.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:01.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:01.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_20 2026-02-20T21:31:01.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:31:01.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:31:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:31:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:31:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:01.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:01.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:31:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:31:01.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:31:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:01.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_20 2026-02-20T21:31:02.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:31:02.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:02.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:02.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_20 2026-02-20T21:31:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:31:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:31:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:31:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:31:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:02.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:02.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:31:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:31:02.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:31:02.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:02.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:02.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:02.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_20 2026-02-20T21:31:02.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_20'] 2026-02-20T21:31:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:02.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_20 2026-02-20T21:31:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:31:02.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:31:02.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:31:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:31:02.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_20 2026-02-20T21:31:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:02.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_20 2026-02-20T21:31:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:31:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '' 2026-02-20T21:31:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:31:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-20T21:31:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-20T21:31:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:31:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-20T21:31:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:31:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:31:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:31:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39122], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:31:02.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39122], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:31:02.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:31:02.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:31:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_20 2026-02-20T21:31:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:31:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_20 '..*' 2026-02-20T21:31:02.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:31:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_20.status 2026-02-20T21:31:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_20.status 2026-02-20T21:31:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:31:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_20 2026-02-20T21:31:02.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:31:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:31:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:31:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39122], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:31:02.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=38, tag_tid=1, entry_tid=39122], [object_number=39, tag_tid=1, entry_tid=39119]]], state=connected] 2026-02-20T21:31:02.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.status 2026-02-20T21:31:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:31:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:31:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:02.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_20 snap 2026-02-20T21:31:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:31:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:31:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:31:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:31:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:31:03.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_20@snap 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_20': 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 121290bf77fd 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.121290bf77fd 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:22:16 2026 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:22:16 2026 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:23:17 2026 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 121290bf77fd 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0d4adb74-259e-47f6-ade8-aab4a76364a5 2026-02-20T21:31:03.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:31:03.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:03.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_20 snap 2026-02-20T21:31:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:31:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:31:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:31:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.export 2026-02-20T21:31:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_20.export 2026-02-20T21:31:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_20.export 2026-02-20T21:31:03.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_20@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.export 2026-02-20T21:31:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:31:03.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_20@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_20.export 2026-02-20T21:31:03.386 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:31:03.382+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:31:03.387 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:31:03.383+0000 7f41304d3640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:31:03.388 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:31:03.384+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:31:03.388 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:31:03.384+0000 7f412cccc640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:31:03.388 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:31:03.384+0000 7f412a4c7640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:31:03.389 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:31:03.384+0000 7f412e4cf640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:31:03.455 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:31:03.451+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:31:03.455 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:31:03.451+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:31:03.456 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:31:03.452+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:31:03.456 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:31:03.452+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:31:03.457 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:31:03.453+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:31:03.457 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:31:03.453+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:31:03.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:31:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_20.export 2026-02-20T21:31:03.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_20.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_20.export 2026-02-20T21:31:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:31:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_21 2026-02-20T21:31:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_21 snap 2026-02-20T21:31:03.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:31:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:31:03.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_21@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:31:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_21 2026-02-20T21:31:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_21 started 2026-02-20T21:31:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:31:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:31:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:05.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:31:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_21 started 2026-02-20T21:31:06.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:31:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:31:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:31:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_21 2026-02-20T21:31:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:31:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:31:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:31:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:31:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:31:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:31:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:31:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_21 2026-02-20T21:31:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:31:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:31:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:06.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:06.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:31:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:31:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:31:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:31:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:31:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:31:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:31:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:31:06.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:31:06.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:31:06.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:31:06.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_21 2026-02-20T21:31:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:31:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:31:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:31:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:31:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:31:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:31:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:31:06.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_21 2026-02-20T21:31:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:31:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:31:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:31:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:31:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:31:06.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_21 2026-02-20T21:31:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:31:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:31:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:31:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:31:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:31:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:31:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:31:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:31:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-20T21:31:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:06.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-20T21:31:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-20T21:31:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-20T21:31:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:06.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:06.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-20T21:31:06.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:06.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:06.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:06.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-20T21:31:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-20T21:31:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:06.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-20T21:31:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:06.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:06.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:06.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:06.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:06.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:06.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:06.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17278], [object_number=17, tag_tid=1, entry_tid=17277], [object_number=16, tag_tid=1, entry_tid=17276], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:06.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:06.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-20T21:31:06.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-20T21:31:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:06.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:06.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:06.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17278], [object_number=17, tag_tid=1, entry_tid=17277], [object_number=16, tag_tid=1, entry_tid=17276], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:06.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17278' '!=' '' 2026-02-20T21:31:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:31:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17278' = '' 2026-02-20T21:31:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:31:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:31:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:31:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:31:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:31:06.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:31:06.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:06.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:31:06.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:31:06.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:31:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:31:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:31:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=17278 2026-02-20T21:31:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 17278 2026-02-20T21:31:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:31:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:31:06.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-20T21:31:06.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:06.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-20T21:31:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-20T21:31:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-20T21:31:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:06.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:06.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-20T21:31:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:06.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:06.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:06.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:06.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:06.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-20T21:31:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-20T21:31:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-20T21:31:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:06.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:06.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:06.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:06.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:06.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:06.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:06.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17278], [object_number=17, tag_tid=1, entry_tid=17277], [object_number=16, tag_tid=1, entry_tid=17276], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:06.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:06.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-20T21:31:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-20T21:31:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17278], [object_number=17, tag_tid=1, entry_tid=17277], [object_number=16, tag_tid=1, entry_tid=17276], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:06.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17278' '!=' 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:06.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:31:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-20T21:31:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-20T21:31:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-20T21:31:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-20T21:31:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-20T21:31:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:07.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:07.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:07.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:07.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-20T21:31:07.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:07.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-20T21:31:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-20T21:31:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:07.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17278], [object_number=17, tag_tid=1, entry_tid=17277], [object_number=16, tag_tid=1, entry_tid=17276], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:07.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:07.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:07.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-20T21:31:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-20T21:31:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17278], [object_number=17, tag_tid=1, entry_tid=17277], [object_number=16, tag_tid=1, entry_tid=17276], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:07.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:07.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17278' '!=' 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:07.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:31:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-20T21:31:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-20T21:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-20T21:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-20T21:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:08.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:08.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-20T21:31:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:08.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:08.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:08.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:08.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:08.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:08.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:08.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:08.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:08.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:08.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:08.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-20T21:31:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:08.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-20T21:31:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-20T21:31:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:08.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:08.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:08.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:08.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:08.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:08.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:08.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17278], [object_number=17, tag_tid=1, entry_tid=17277], [object_number=16, tag_tid=1, entry_tid=17276], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:08.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:08.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-20T21:31:08.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-20T21:31:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:08.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:08.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:08.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:08.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:08.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:08.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:08.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17278], [object_number=17, tag_tid=1, entry_tid=17277], [object_number=16, tag_tid=1, entry_tid=17276], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:08.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17278' '!=' 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:08.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:08.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:09.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:09.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:09.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:09.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:09.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:09.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:09.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:09.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-20T21:31:09.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:09.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-20T21:31:09.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:09.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:09.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-20T21:31:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:09.904 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:09.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:09.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:09.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17278], [object_number=17, tag_tid=1, entry_tid=17277], [object_number=16, tag_tid=1, entry_tid=17276], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:09.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:09.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-20T21:31:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-20T21:31:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:09.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17278], [object_number=17, tag_tid=1, entry_tid=17277], [object_number=16, tag_tid=1, entry_tid=17276], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:09.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17278' '!=' 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:09.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:31:11.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-20T21:31:11.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-20T21:31:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:11.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:11.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:11.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:11.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:11.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:11.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:11.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-20T21:31:12.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-20T21:31:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-20T21:31:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:12.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:12.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:12.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:12.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:12.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:12.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:12.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:12.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17278], [object_number=17, tag_tid=1, entry_tid=17277], [object_number=16, tag_tid=1, entry_tid=17276], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:12.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:12.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-20T21:31:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-20T21:31:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:12.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:12.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:12.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:12.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:12.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:12.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:12.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:12.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=18, tag_tid=1, entry_tid=17278], [object_number=17, tag_tid=1, entry_tid=17277], [object_number=16, tag_tid=1, entry_tid=17276], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:12.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=18, tag_tid=1, entry_tid=17278' '!=' 'object_number=18, tag_tid=1, entry_tid=17278' 2026-02-20T21:31:12.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:12.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:31:14.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_21 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_21 ']' 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_21' 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_21 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:14.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_21 2026-02-20T21:31:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:14.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:14.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:14.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:14.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:14.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_21 2026-02-20T21:31:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_21 2026-02-20T21:31:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '' 2026-02-20T21:31:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:14.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:14.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:14.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_21 2026-02-20T21:31:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:14.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_21 '..*' 2026-02-20T21:31:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:31:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_21.status 2026-02-20T21:31:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_21.status 2026-02-20T21:31:14.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:14.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:14.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_21 2026-02-20T21:31:14.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:14.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=18, tag_tid=1, entry_tid=17278], [object_number=19, tag_tid=1, entry_tid=17275]]], state=connected] 2026-02-20T21:31:14.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.status 2026-02-20T21:31:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_21 snap 2026-02-20T21:31:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:14.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:14.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:14.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:31:14.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:31:14.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:31:14.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:31:15.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_21@snap 2026-02-20T21:31:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_21': 2026-02-20T21:31:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:31:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:31:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:31:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ec7cadf4de 2026-02-20T21:31:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ec7cadf4de 2026-02-20T21:31:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:31:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:31:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:31:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:31:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:22:27 2026 2026-02-20T21:31:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:22:27 2026 2026-02-20T21:31:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:23:28 2026 2026-02-20T21:31:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:31:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ec7cadf4de 2026-02-20T21:31:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:31:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:31:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: f7d9d1b1-323f-437c-a5ae-cf547b016e83 2026-02-20T21:31:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:31:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_21 snap 2026-02-20T21:31:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:31:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:31:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:31:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.export 2026-02-20T21:31:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_21.export 2026-02-20T21:31:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_21.export 2026-02-20T21:31:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_21@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.export 2026-02-20T21:31:15.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:31:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_21@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_21.export 2026-02-20T21:31:15.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:31:15.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_21.export 2026-02-20T21:31:15.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_21.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_21.export 2026-02-20T21:31:15.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:31:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_22 2026-02-20T21:31:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_22 snap 2026-02-20T21:31:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:31:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:31:15.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_22@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:31:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_22 2026-02-20T21:31:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_22 started 2026-02-20T21:31:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:31:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:31:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:31:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_22 started 2026-02-20T21:31:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:31:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:31:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:31:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_22 2026-02-20T21:31:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:31:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:31:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:31:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:31:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:31:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:31:18.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:31:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_22 2026-02-20T21:31:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:31:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:31:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:31:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:31:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:31:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:31:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:31:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:31:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:31:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:31:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:31:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:18.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_22 2026-02-20T21:31:18.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:31:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:31:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:31:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:31:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:31:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:31:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:31:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_22 2026-02-20T21:31:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:31:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:31:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:31:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:31:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:31:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_22 2026-02-20T21:31:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:31:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:31:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:31:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:31:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:31:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:31:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:31:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:31:18.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-20T21:31:18.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:18.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:18.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:18.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:18.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-20T21:31:18.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-20T21:31:18.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-20T21:31:18.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:18.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:18.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:18.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:18.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:18.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:18.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:18.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:18.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:18.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:18.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:18.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:18.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:18.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:18.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:18.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:18.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:18.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-20T21:31:18.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:18.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:18.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:18.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-20T21:31:18.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:18.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:18.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:18.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:18.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:18.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:18.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:18.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:18.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:18.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:18.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:18.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-20T21:31:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:18.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-20T21:31:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:18.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:18.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:18.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:18.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:18.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:18.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:18.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:18.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:18.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19049' '!=' '' 2026-02-20T21:31:18.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:31:18.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19049' = '' 2026-02-20T21:31:18.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:31:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:31:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:31:18.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:31:18.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:31:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:31:18.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:31:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:31:18.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:31:18.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:18.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:31:18.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:31:18.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=19049 2026-02-20T21:31:18.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 19049 2026-02-20T21:31:18.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:31:18.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:18.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:31:18.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-20T21:31:18.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:18.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:18.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:18.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:18.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-20T21:31:18.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-20T21:31:18.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-20T21:31:18.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:18.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:18.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:18.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:18.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:18.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:18.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:18.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:18.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:18.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:18.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:18.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:18.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:18.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:18.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:18.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:18.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:18.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:18.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:18.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:18.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:18.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:18.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:18.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:18.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:18.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:18.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:18.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:18.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:18.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:18.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:18.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:18.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:18.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-20T21:31:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-20T21:31:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:18.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:18.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:18.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:18.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-20T21:31:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-20T21:31:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:18.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:18.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:18.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:18.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19049' '!=' 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:18.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:31:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-20T21:31:19.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:19.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:19.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:19.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:19.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-20T21:31:19.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-20T21:31:19.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-20T21:31:19.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:19.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:19.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:19.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:19.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:19.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:19.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:19.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:19.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:19.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-20T21:31:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:19.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-20T21:31:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:19.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:19.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:19.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:19.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-20T21:31:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:19.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-20T21:31:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:19.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:19.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:19.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:19.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19049' '!=' 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:31:20.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-20T21:31:20.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:20.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:20.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:20.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:20.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-20T21:31:20.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-20T21:31:20.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-20T21:31:20.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:20.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:20.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:20.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:20.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:20.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:20.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:20.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-20T21:31:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-20T21:31:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:20.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:20.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:20.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:20.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:20.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:20.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:20.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:20.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:20.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-20T21:31:20.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:20.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:20.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:20.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-20T21:31:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:20.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:20.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:20.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:20.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:20.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:20.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:20.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:20.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:20.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:20.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:20.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19049' '!=' 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:20.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:20.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:31:21.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-20T21:31:21.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:21.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:21.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:21.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:21.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-20T21:31:21.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-20T21:31:21.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-20T21:31:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:21.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:21.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:21.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:21.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:21.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:21.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:21.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:21.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:21.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:21.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:21.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:21.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:21.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:21.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:21.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:21.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:21.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:21.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:21.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:21.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:21.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:21.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-20T21:31:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-20T21:31:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:21.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:21.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:21.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:21.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-20T21:31:21.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:21.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-20T21:31:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:21.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:21.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:21.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:21.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:21.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:21.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:21.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19049' '!=' 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:21.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:21.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:31:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-20T21:31:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:23.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-20T21:31:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-20T21:31:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-20T21:31:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:23.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:23.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:23.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:23.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:23.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:23.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:23.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:24.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:24.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-20T21:31:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-20T21:31:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:24.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:24.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:24.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-20T21:31:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-20T21:31:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:24.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:24.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:24.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:24.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:24.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:24.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:24.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:24.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:24.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:24.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19049' '!=' 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:24.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:24.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:31:25.608 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-20T21:31:25.609 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-20T21:31:25.609 INFO:tasks.rbd_mirror_thrash:waiting for 5.23037641058396 secs before reviving daemons 2026-02-20T21:31:25.609 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:31:25.607+0000 7f5354451640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 84053) UID: 1000 2026-02-20T21:31:26.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-20T21:31:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:31:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:31:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:31:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-20T21:31:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-20T21:31:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-20T21:31:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:31:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:31:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:31:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:31:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:31:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:26.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:26.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:26.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:26.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:26.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:26.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:26.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:26.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:26.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-20T21:31:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-20T21:31:26.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:31:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:31:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:31:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:31:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:26.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:31:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:31:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:31:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:31:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:31:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:26.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:31:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:31:26.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:31:26.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:26.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:26.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:26.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2026-02-20T21:31:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:31:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:31:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:31:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:31:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:31:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:31:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:31:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:26.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:31:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:31:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:31:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:31:26.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:31:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:26.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:26.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:26.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:26.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:26.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_22 2026-02-20T21:31:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:31:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:31:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:31:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:31:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:31:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:31:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:31:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:31:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:31:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:31:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:31:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:31:26.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:26.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:26.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:26.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:26.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:26.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:26.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:26.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:26.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:31:27.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:27.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:27.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:27.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:27.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:27.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:27.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:27.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:27.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:27.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:27.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:27.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:27.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-20T21:31:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-20T21:31:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:31:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:31:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:31:27.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:27.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:27.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:27.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:31:27.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:27.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:27.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:27.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:31:27.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:27.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:31:27.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:31:27.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:31:27.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:31:27.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:27.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:27.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:27.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:27.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:27.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:27.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:27.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2026-02-20T21:31:27.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:31:27.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:27.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:27.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:31:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:31:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:31:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:31:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:31:27.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:31:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:31:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:31:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:27.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:31:27.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:31:27.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:31:27.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:27.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:27.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:27.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:27.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:27.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_22 2026-02-20T21:31:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:31:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:31:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:31:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:27.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:27.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:31:27.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:27.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:27.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:27.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:31:27.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:27.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:31:27.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:31:27.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:27.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:27.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:31:27.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:31:27.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:27.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:27.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:27.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:27.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:27.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:27.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:27.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:31:27.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:31:27.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:31:27.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:27.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:27.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:27.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:27.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:27.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:31:29.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:29.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:29.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:29.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:29.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:29.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-20T21:31:29.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:29.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-20T21:31:29.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:31:29.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:29.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:29.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:31:29.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:31:29.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:29.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:29.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:29.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:31:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:31:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:31:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:31:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:31:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:31:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:31:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:31:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:31:29.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:29.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:29.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:29.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:29.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2026-02-20T21:31:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:31:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:29.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:29.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:31:29.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:31:29.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:29.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:29.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:29.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:31:29.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:29.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:29.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:29.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:31:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:31:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:31:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:31:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:31:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:29.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:29.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:29.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:29.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:31:29.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:31:29.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:31:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:29.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:29.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:29.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_22 2026-02-20T21:31:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:31:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:31:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:31:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:31:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:31:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:31:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:31:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:31:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:31:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:29.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:31:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:31:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:31:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:29.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:29.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:29.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:29.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:29.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-20T21:31:30.840 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-20T21:31:30.840 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:31:30.840 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-20T21:31:30.841 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-20T21:31:30.841 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-20T21:31:30.843 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-20T21:31:30.843 INFO:tasks.rbd_mirror_thrash:waiting for 105.74745400865973 secs before thrashing 2026-02-20T21:31:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:33.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:33.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:33.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:33.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:33.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:33.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:33.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:33.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:33.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:33.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:33.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:33.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:33.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:33.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:33.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:33.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:33.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:33.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:33.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:33.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-20T21:31:33.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:31:33.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:33.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:33.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:31:33.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:31:33.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:33.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:31:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:31:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:31:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:31:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:31:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:31:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:31:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:31:33.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:31:33.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:33.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:33.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:33.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:33.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:33.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:33.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:33.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2026-02-20T21:31:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:31:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:31:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:31:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:33.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:31:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:31:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:31:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:31:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:31:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:31:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:31:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:31:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:31:33.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:33.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:33.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_22 2026-02-20T21:31:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:31:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:31:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:31:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:31:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:31:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:31:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:31:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:31:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:31:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:31:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:31:33.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:31:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 8 2026-02-20T21:31:42.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:42.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:42.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:42.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:42.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:42.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:42.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:42.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:42.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:42.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-20T21:31:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:31:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:31:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:31:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:31:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:31:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:31:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:31:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:31:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:31:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:31:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:31:42.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:31:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:42.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:42.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:42.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2026-02-20T21:31:42.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:31:42.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:42.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:31:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:31:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:31:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:31:42.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:31:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:31:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:31:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:31:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:42.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:31:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:31:42.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:31:42.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:42.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:42.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:42.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_22 2026-02-20T21:31:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:31:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:31:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:31:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:31:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:31:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:42.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:31:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:31:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:31:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:31:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:42.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:42.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:31:42.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:31:42.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:31:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:42.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:42.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:42.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:42.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:42.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:42.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 8 2026-02-20T21:31:50.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:50.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:50.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:50.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:50.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:50.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:50.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:50.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:50.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:50.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:50.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:50.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:50.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:50.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-20T21:31:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:31:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:31:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:31:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:50.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:31:50.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:31:50.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:50.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:50.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:50.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:50.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_22 2026-02-20T21:31:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:31:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:31:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:31:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:50.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:31:50.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:31:50.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:50.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:50.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:31:50.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:50.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_22 2026-02-20T21:31:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:31:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:31:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:31:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:31:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:31:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:31:50.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:31:50.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:50.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:31:50.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 8 2026-02-20T21:31:53.413 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:31:53.411+0000 7faa4d82c640 -1 rbd::mirror::image_replayer::journal::Replayer: 0x561ec4158700 get_replay_status: replay not running 2026-02-20T21:31:58.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:31:58.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:31:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:31:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:31:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:31:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:31:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:58.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:31:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:31:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:31:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:31:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:58.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:31:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:31:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:31:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:31:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:31:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:31:58.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:31:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:31:58.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:31:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:31:58.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-20T21:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:31:58.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:31:58.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:31:58.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:58.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:58.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:31:58.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-20T21:31:58.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:31:58.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-20T21:31:58.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:58.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:58.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:58.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-20T21:31:58.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:58.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:58.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:58.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:31:58.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:58.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:58.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:58.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:58.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:58.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:58.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:58.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:58.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:58.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:58.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:58.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:58.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:31:58.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-20T21:31:58.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:58.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:58.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:58.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-20T21:31:58.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:31:58.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:31:58.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:31:58.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:31:58.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:31:58.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:31:58.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:31:58.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:58.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:31:58.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:58.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:31:58.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:31:58.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:31:58.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:58.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:31:58.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:31:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19049' '!=' 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:31:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:31:58.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-20T21:32:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-20T21:32:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:02.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:32:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-20T21:32:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-20T21:32:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-20T21:32:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:02.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:02.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:32:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:02.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:02.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:02.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:02.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:02.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:32:02.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:32:02.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:02.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-20T21:32:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:02.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:02.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:02.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:02.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-20T21:32:02.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:02.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-20T21:32:02.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:02.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:32:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-20T21:32:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:32:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:32:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:32:02.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:32:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:32:02.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:32:02.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:32:02.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:32:02.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:32:02.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:02.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-20T21:32:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:32:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-20T21:32:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:32:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:32:02.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:32:02.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:02.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:32:02.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:32:02.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:32:02.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:02.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:02.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:02.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:32:02.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=1, entry_tid=19049], [object_number=16, tag_tid=1, entry_tid=19048], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:32:02.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:32:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:32:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:32:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=17, tag_tid=1, entry_tid=19049' '!=' 'object_number=17, tag_tid=1, entry_tid=19049' 2026-02-20T21:32:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:02.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-20T21:32:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_22 2026-02-20T21:32:06.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:32:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_22 ']' 2026-02-20T21:32:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_22' 2026-02-20T21:32:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_22 2026-02-20T21:32:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:06.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:06.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:06.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:06.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_22 2026-02-20T21:32:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:06.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:06.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:06.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:06.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_22 2026-02-20T21:32:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_22'] 2026-02-20T21:32:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:06.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:06.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_22 2026-02-20T21:32:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:06.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_22 2026-02-20T21:32:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_22 2026-02-20T21:32:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:32:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '' 2026-02-20T21:32:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:32:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:32:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:32:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:32:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:32:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:32:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:32:07.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:32:07.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:32:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:07.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_22 2026-02-20T21:32:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:32:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_22 '..*' 2026-02-20T21:32:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:32:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_22.status 2026-02-20T21:32:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_22.status 2026-02-20T21:32:07.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:07.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:32:07.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_22 2026-02-20T21:32:07.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:32:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:32:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=19047], [object_number=18, tag_tid=1, entry_tid=19046]]], state=connected] 2026-02-20T21:32:07.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.status 2026-02-20T21:32:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_22 snap 2026-02-20T21:32:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:32:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:32:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:32:07.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:32:08.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_22@snap 2026-02-20T21:32:08.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_22': 2026-02-20T21:32:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:32:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:32:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:32:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ff5dabbe3e 2026-02-20T21:32:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ff5dabbe3e 2026-02-20T21:32:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:32:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:32:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:32:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:32:08.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:22:32 2026 2026-02-20T21:32:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:22:32 2026 2026-02-20T21:32:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:22:32 2026 2026-02-20T21:32:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:32:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ff5dabbe3e 2026-02-20T21:32:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:32:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:32:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: e5e4499d-9f2d-4424-96b8-4076c9d740c0 2026-02-20T21:32:08.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:32:08.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_22 snap 2026-02-20T21:32:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:32:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:32:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.export 2026-02-20T21:32:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_22.export 2026-02-20T21:32:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_22.export 2026-02-20T21:32:08.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_22@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.export 2026-02-20T21:32:08.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:32:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_22@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_22.export 2026-02-20T21:32:08.249 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:08.245+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:32:08.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:08.245+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:32:08.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:08.245+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:32:08.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:08.246+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:32:08.250 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:08.246+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:32:08.288 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:08.284+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:32:08.289 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:08.285+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:32:08.289 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:08.285+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:32:08.290 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:08.286+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:32:08.290 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:08.286+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:32:08.294 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:08.290+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:32:08.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:32:08.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_22.export 2026-02-20T21:32:08.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_22.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_22.export 2026-02-20T21:32:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:32:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_23 2026-02-20T21:32:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_23 snap 2026-02-20T21:32:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:08.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_23@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:32:09.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_23 2026-02-20T21:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_23 started 2026-02-20T21:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:09.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:32:10.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_23 started 2026-02-20T21:32:10.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:10.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:10.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:10.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:32:10.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:32:10.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:32:10.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_23 2026-02-20T21:32:10.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:32:10.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:32:10.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:10.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:10.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:32:10.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:32:10.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:10.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:32:10.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:32:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_23 2026-02-20T21:32:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:32:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:10.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:32:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:32:10.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:10.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:10.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:10.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:10.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:10.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:10.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:10.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:32:10.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:32:10.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:10.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:10.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_23 2026-02-20T21:32:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:32:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_23 2026-02-20T21:32:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:32:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:32:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:32:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:10.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:32:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:32:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:10.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:10.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:10.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:10.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_23 2026-02-20T21:32:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:32:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_23 2026-02-20T21:32:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:32:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:32:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:32:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:32:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:32:10.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:10.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:10.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:10.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:10.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_23 2026-02-20T21:32:10.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:10.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:32:10.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_23 2026-02-20T21:32:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:32:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:32:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:32:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:32:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:32:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:32:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:32:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:32:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:32:10.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:32:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:32:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:32:10.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:32:10.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_23 2026-02-20T21:32:10.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:32:10.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:32:10.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:32:10.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:32:10.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:32:10.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:32:10.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:32:10.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:10.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_23 2026-02-20T21:32:10.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:32:10.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:10.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:32:10.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:32:10.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:10.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:32:10.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_23 2026-02-20T21:32:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:32:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:32:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:32:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:32:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:32:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:32:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:10.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:32:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_23 2026-02-20T21:32:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_23 ']' 2026-02-20T21:32:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-20T21:32:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-20T21:32:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-20T21:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:10.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:10.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:10.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:10.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:11.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:11.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2026-02-20T21:32:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:11.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:11.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:11.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:11.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:11.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:11.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:11.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:11.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_23 2026-02-20T21:32:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:11.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:11.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:11.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:11.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:11.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_23 2026-02-20T21:32:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:32:11.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:32:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:32:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:32:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:11.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:32:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:32:11.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:32:11.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:11.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:11.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:11.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:11.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-20T21:32:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-20T21:32:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:11.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:32:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-20T21:32:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:11.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:11.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-20T21:32:11.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=42094], [object_number=41, tag_tid=1, entry_tid=42093], [object_number=40, tag_tid=1, entry_tid=42092], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:11.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:11.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-20T21:32:11.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:11.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:11.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:11.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-20T21:32:11.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:11.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:11.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:11.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:11.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:32:11.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-20T21:32:11.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:11.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:11.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-20T21:32:11.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:11.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=42094], [object_number=41, tag_tid=1, entry_tid=42093], [object_number=40, tag_tid=1, entry_tid=42092], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:11.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:11.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:11.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:11.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=42094' '!=' '' 2026-02-20T21:32:11.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:32:11.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=42094' = '' 2026-02-20T21:32:11.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:11.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:11.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:32:11.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:11.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:32:11.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:11.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:32:11.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:11.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:32:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:32:11.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:11.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:32:11.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:11.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:32:11.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:11.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42094 2026-02-20T21:32:11.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 42094 2026-02-20T21:32:11.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:32:11.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:11.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:32:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_23 2026-02-20T21:32:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:11.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_23 ']' 2026-02-20T21:32:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-20T21:32:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-20T21:32:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:11.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:11.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-20T21:32:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:11.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:11.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:11.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:11.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:11.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:11.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:11.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:11.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:11.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:11.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2026-02-20T21:32:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:11.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:11.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:11.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:11.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:11.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:11.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:11.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:11.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_23 2026-02-20T21:32:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:11.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:11.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:11.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:11.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:11.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:11.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:11.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:11.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:11.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:11.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:11.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:11.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:11.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:11.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:11.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:11.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:11.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:11.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:11.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:11.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:11.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:11.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:11.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_23 2026-02-20T21:32:11.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:32:11.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:11.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:32:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:32:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:32:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:32:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:11.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:11.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:32:11.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:32:11.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:32:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:11.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:11.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:11.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:11.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:11.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-20T21:32:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-20T21:32:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:32:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-20T21:32:11.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:11.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:11.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-20T21:32:11.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=42094], [object_number=41, tag_tid=1, entry_tid=42093], [object_number=40, tag_tid=1, entry_tid=42092], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:11.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:11.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-20T21:32:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-20T21:32:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:32:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-20T21:32:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-20T21:32:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:11.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:11.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:11.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:11.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:11.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=42094], [object_number=41, tag_tid=1, entry_tid=42093], [object_number=40, tag_tid=1, entry_tid=42092], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:11.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=42094' '!=' 'object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:32:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_23 2026-02-20T21:32:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:12.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_23 ']' 2026-02-20T21:32:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-20T21:32:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-20T21:32:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:12.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:12.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:12.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:12.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-20T21:32:12.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:12.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:12.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:12.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:12.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:12.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:12.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:12.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:12.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:12.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:12.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:12.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:12.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:12.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:12.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:12.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2026-02-20T21:32:12.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:12.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:12.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:12.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:12.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:12.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:12.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:12.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:12.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:12.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:12.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:12.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:12.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:12.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:12.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:12.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:12.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_23 2026-02-20T21:32:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:12.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:12.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:12.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:12.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:12.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:12.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:12.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:12.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:12.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:12.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:12.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:12.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:12.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:12.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:12.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:12.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:12.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:12.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:12.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_23 2026-02-20T21:32:12.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:32:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:32:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:32:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:32:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:12.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:32:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:32:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:32:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:32:12.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:32:12.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:32:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-20T21:32:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-20T21:32:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:32:12.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-20T21:32:12.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:12.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:12.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-20T21:32:12.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:12.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:12.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:12.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:12.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:12.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=42094], [object_number=41, tag_tid=1, entry_tid=42093], [object_number=40, tag_tid=1, entry_tid=42092], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:12.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:12.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-20T21:32:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-20T21:32:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:12.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:12.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:12.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:12.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:32:12.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-20T21:32:12.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:12.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:12.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-20T21:32:12.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:12.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:12.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:12.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:12.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:12.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=42094], [object_number=41, tag_tid=1, entry_tid=42093], [object_number=40, tag_tid=1, entry_tid=42092], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:12.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=42094' '!=' 'object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:12.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:32:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_23 2026-02-20T21:32:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_23 ']' 2026-02-20T21:32:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-20T21:32:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-20T21:32:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:13.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:13.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:13.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:13.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-20T21:32:13.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:13.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:13.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:13.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:13.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:13.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:13.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:13.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:13.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:13.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:13.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:13.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:13.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:13.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:13.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:13.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:13.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:13.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:13.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:13.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:13.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:13.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:13.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:13.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2026-02-20T21:32:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:13.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:13.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:13.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_23 2026-02-20T21:32:13.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:13.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:13.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:13.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:13.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:13.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:13.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:13.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:13.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:13.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:13.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:13.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:13.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:13.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:13.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:13.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:13.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:13.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:13.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:13.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:13.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:13.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:13.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:13.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:13.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:13.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:13.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_23 2026-02-20T21:32:13.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:32:13.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:13.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:13.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:32:13.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:32:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:32:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:32:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:32:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:32:13.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:32:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:32:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:32:13.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:32:13.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:32:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:13.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:13.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-20T21:32:13.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:13.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:13.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:13.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-20T21:32:13.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:13.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:13.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:13.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:13.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:32:13.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-20T21:32:13.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:13.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:13.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-20T21:32:13.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:13.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=42094], [object_number=41, tag_tid=1, entry_tid=42093], [object_number=40, tag_tid=1, entry_tid=42092], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:13.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:13.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:13.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-20T21:32:13.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:13.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:13.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:13.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-20T21:32:13.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:13.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:13.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:13.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:13.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:32:13.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-20T21:32:13.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:13.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:13.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-20T21:32:13.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:13.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=42094], [object_number=41, tag_tid=1, entry_tid=42093], [object_number=40, tag_tid=1, entry_tid=42092], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:13.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:13.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:13.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:13.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=42094' '!=' 'object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:13.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:13.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_23 2026-02-20T21:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_23 ']' 2026-02-20T21:32:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-20T21:32:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-20T21:32:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:15.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:15.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-20T21:32:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:15.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:15.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:15.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:15.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:15.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:15.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:15.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2026-02-20T21:32:15.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:15.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:15.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:15.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:15.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:15.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:15.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:15.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_23 2026-02-20T21:32:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:15.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:15.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:15.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_23 2026-02-20T21:32:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:32:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:32:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:32:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:15.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:32:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:32:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:32:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:32:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:32:15.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:32:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:32:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:32:15.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:32:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-20T21:32:15.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:15.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:15.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:15.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-20T21:32:15.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:15.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:15.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:15.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:15.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:32:15.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-20T21:32:15.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:15.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:15.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-20T21:32:15.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:15.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=42094], [object_number=41, tag_tid=1, entry_tid=42093], [object_number=40, tag_tid=1, entry_tid=42092], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:15.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:15.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:15.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-20T21:32:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-20T21:32:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:32:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-20T21:32:15.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-20T21:32:15.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:15.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:15.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:15.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:15.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:15.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=42094], [object_number=41, tag_tid=1, entry_tid=42093], [object_number=40, tag_tid=1, entry_tid=42092], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=42094' '!=' 'object_number=42, tag_tid=1, entry_tid=42094' 2026-02-20T21:32:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:15.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:32:17.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_23 2026-02-20T21:32:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_23 ']' 2026-02-20T21:32:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_23' 2026-02-20T21:32:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_23 2026-02-20T21:32:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:17.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:17.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_23 2026-02-20T21:32:17.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:17.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:17.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:17.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:17.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:17.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:17.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:17.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:17.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:17.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:17.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:17.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:17.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:17.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:17.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:17.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:17.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:17.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:17.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:17.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:17.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:17.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:17.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:17.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:17.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_23 2026-02-20T21:32:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:17.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:17.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:17.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:17.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:17.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:17.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_23 2026-02-20T21:32:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:17.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:17.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:17.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:17.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_23'] 2026-02-20T21:32:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_23 2026-02-20T21:32:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:32:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:32:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:32:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:32:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:32:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:32:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:32:17.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:32:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:32:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:32:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:32:17.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:32:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:32:17.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:32:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:32:17.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_23 2026-02-20T21:32:17.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_23 2026-02-20T21:32:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '' 2026-02-20T21:32:17.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:32:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-20T21:32:17.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:17.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:17.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-20T21:32:17.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:17.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:17.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:17.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:17.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_23 2026-02-20T21:32:17.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_23 '..*' 2026-02-20T21:32:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:32:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_23.status 2026-02-20T21:32:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_23.status 2026-02-20T21:32:17.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_23 2026-02-20T21:32:17.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:17.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:17.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:17.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:17.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:17.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=42094], [object_number=43, tag_tid=1, entry_tid=42091]]], state=connected] 2026-02-20T21:32:17.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.status 2026-02-20T21:32:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_23 snap 2026-02-20T21:32:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:17.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:17.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:32:17.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:32:17.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:32:18.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_23@snap 2026-02-20T21:32:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_23': 2026-02-20T21:32:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:32:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:32:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:32:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11b3ac204b17 2026-02-20T21:32:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11b3ac204b17 2026-02-20T21:32:18.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:32:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:32:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:32:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:32:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:22:38 2026 2026-02-20T21:32:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:22:38 2026 2026-02-20T21:32:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:24:40 2026 2026-02-20T21:32:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:32:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11b3ac204b17 2026-02-20T21:32:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:32:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:32:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: df1ee7cb-a275-4810-a91a-ae5f44bf0c61 2026-02-20T21:32:18.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:32:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_23 snap 2026-02-20T21:32:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:32:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:32:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.export 2026-02-20T21:32:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_23.export 2026-02-20T21:32:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_23.export 2026-02-20T21:32:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_23@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.export 2026-02-20T21:32:18.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:32:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_23@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_23.export 2026-02-20T21:32:19.054 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:19.050+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:32:19.055 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:19.051+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:32:19.055 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:19.051+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:32:19.056 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:19.051+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:32:19.112 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:32:19.108+0000 7fc9bf00a640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 3 2026-02-20T21:32:19.113 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:32:19.108+0000 7fc9c3012640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 2 2026-02-20T21:32:19.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:32:19.109+0000 7fc9c2811640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 3 2026-02-20T21:32:19.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:32:19.109+0000 7fc9c000c640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 4 2026-02-20T21:32:19.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:32:19.109+0000 7fc9bc004640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 4 2026-02-20T21:32:19.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:32:19.109+0000 7fc9c080d640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 4 2026-02-20T21:32:19.114 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:32:19.109+0000 7fc9c2010640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 4 2026-02-20T21:32:19.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:32:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_23.export 2026-02-20T21:32:19.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_23.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_23.export 2026-02-20T21:32:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:32:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_24 2026-02-20T21:32:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_24 snap 2026-02-20T21:32:19.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:19.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:19.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:32:19.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:19.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_24@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:32:20.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_24 2026-02-20T21:32:20.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:20.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:20.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:32:20.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_24 started 2026-02-20T21:32:20.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:20.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:20.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:32:20.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:32:20.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:32:20.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:20.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:32:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_24 started 2026-02-20T21:32:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:32:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:32:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:32:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:32:21.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_24 2026-02-20T21:32:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:32:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:32:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:32:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:32:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:21.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:32:21.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:21.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:32:21.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_24 2026-02-20T21:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:21.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:21.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:32:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:32:21.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:21.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:21.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_24 2026-02-20T21:32:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-20T21:32:21.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:32:21.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:21.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:21.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_24 2026-02-20T21:32:21.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:32:21.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:21.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:21.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:21.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:32:21.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:32:21.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:22.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:22.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:22.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:22.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_24 2026-02-20T21:32:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-20T21:32:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:32:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_24 2026-02-20T21:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:22.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:22.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:32:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:32:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:32:22.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_24 2026-02-20T21:32:22.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:32:22.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:32:22.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:32:22.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:32:22.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:32:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:32:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:32:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_24 2026-02-20T21:32:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:32:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:32:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:32:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:32:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:32:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_24 2026-02-20T21:32:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:32:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:32:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:32:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:32:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:32:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:32:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:32:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:22.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:32:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_24 2026-02-20T21:32:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:22.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:32:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_24 ']' 2026-02-20T21:32:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_24' 2026-02-20T21:32:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_24 2026-02-20T21:32:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:22.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:22.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_24 2026-02-20T21:32:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:22.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:22.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:22.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:22.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:22.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_24 2026-02-20T21:32:22.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-20T21:32:22.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:22.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:22.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_24 2026-02-20T21:32:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:22.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:22.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:22.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:22.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:22.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:22.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:22.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_24 2026-02-20T21:32:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_24'] 2026-02-20T21:32:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:22.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:22.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:22.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_24 2026-02-20T21:32:22.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:22.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:22.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:22.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:22.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:22.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:22.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:22.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:22.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:22.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_24 2026-02-20T21:32:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_24 2026-02-20T21:32:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-20T21:32:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '' 2026-02-20T21:32:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-20T21:32:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-20T21:32:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-20T21:32:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_24.status 2026-02-20T21:32:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-20T21:32:22.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_24.status 2026-02-20T21:32:22.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:32:22.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:32:22.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:22.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23547], [object_number=22, tag_tid=1, entry_tid=23546]]], state=connected] 2026-02-20T21:32:22.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23547], [object_number=22, tag_tid=1, entry_tid=23546]]], state=connected] 2026-02-20T21:32:22.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_24.status 2026-02-20T21:32:22.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:32:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_24 2026-02-20T21:32:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:22.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-20T21:32:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_24 '..*' 2026-02-20T21:32:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-20T21:32:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_24.status 2026-02-20T21:32:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_24.status 2026-02-20T21:32:22.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_24.status 2026-02-20T21:32:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_24 2026-02-20T21:32:22.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_24.status 2026-02-20T21:32:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:32:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:32:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23547], [object_number=22, tag_tid=1, entry_tid=23546]]], state=connected] 2026-02-20T21:32:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23547], [object_number=22, tag_tid=1, entry_tid=23546]]], state=connected] 2026-02-20T21:32:22.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_24.status 2026-02-20T21:32:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:32:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:32:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_24 snap 2026-02-20T21:32:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:32:22.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:32:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:32:22.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:32:23.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_24@snap 2026-02-20T21:32:23.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_24': 2026-02-20T21:32:23.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:32:23.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:32:23.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:32:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1222fd38ce73 2026-02-20T21:32:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1222fd38ce73 2026-02-20T21:32:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:32:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:32:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:32:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:32:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:22:49 2026 2026-02-20T21:32:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:22:49 2026 2026-02-20T21:32:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:23:50 2026 2026-02-20T21:32:23.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:32:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1222fd38ce73 2026-02-20T21:32:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:32:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:32:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 50733ee2-660d-4c70-acfb-b62f4cda3626 2026-02-20T21:32:23.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:32:23.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:23.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_24 snap 2026-02-20T21:32:23.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:23.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:32:23.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:23.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:32:23.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_24.export 2026-02-20T21:32:23.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_24.export 2026-02-20T21:32:23.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_24.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_24.export 2026-02-20T21:32:23.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_24@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_24.export 2026-02-20T21:32:23.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:32:23.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_24@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_24.export 2026-02-20T21:32:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:32:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_24.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_24.export 2026-02-20T21:32:23.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_24.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_24.export 2026-02-20T21:32:23.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:23.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:32:23.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_25 2026-02-20T21:32:23.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_25 snap 2026-02-20T21:32:23.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:23.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:23.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:23.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:23.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_25@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:32:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_25 2026-02-20T21:32:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_25 started 2026-02-20T21:32:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:26.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:32:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:32:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:26.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:32:27.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_25 started 2026-02-20T21:32:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:32:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:32:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:32:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_25 2026-02-20T21:32:27.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:32:27.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:32:27.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:27.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:27.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:32:27.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:32:27.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:27.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:32:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:32:27.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_25 2026-02-20T21:32:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:32:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:27.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:32:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:32:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:27.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:32:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:32:27.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:27.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:27.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_25 2026-02-20T21:32:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:32:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:32:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:32:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:32:28.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_25 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_25 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:28.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:32:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:32:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:32:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:28.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:32:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_25 2026-02-20T21:32:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_25 ']' 2026-02-20T21:32:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-20T21:32:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-20T21:32:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:28.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:28.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-20T21:32:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:28.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:28.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:28.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:28.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:28.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:28.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:28.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:28.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:28.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:28.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-20T21:32:28.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-20T21:32:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:28.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-20T21:32:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-20T21:32:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-20T21:32:28.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-20T21:32:28.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:32:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:32:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638]]], state=connected] 2026-02-20T21:32:28.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26640], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638], [object_number=25, tag_tid=1, entry_tid=26637]]], state=connected] 2026-02-20T21:32:28.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:28.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:32:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-20T21:32:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:28.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-20T21:32:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-20T21:32:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-20T21:32:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-20T21:32:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:28.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:32:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:32:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638]]], state=connected] 2026-02-20T21:32:28.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26640], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638], [object_number=25, tag_tid=1, entry_tid=26637]]], state=connected] 2026-02-20T21:32:28.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:28.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=26640' '!=' '' 2026-02-20T21:32:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:32:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=26640' = '' 2026-02-20T21:32:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:32:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:32:28.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:32:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:28.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:32:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:28.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:32:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:32:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:32:28.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:28.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:32:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:32:28.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26640 2026-02-20T21:32:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 26640 2026-02-20T21:32:28.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:32:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:32:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_25 2026-02-20T21:32:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_25 ']' 2026-02-20T21:32:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-20T21:32:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-20T21:32:28.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:28.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-20T21:32:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:28.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:28.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:28.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:28.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:28.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-20T21:32:28.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:28.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-20T21:32:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-20T21:32:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-20T21:32:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-20T21:32:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-20T21:32:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:32:28.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638]]], state=connected] 2026-02-20T21:32:28.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26640], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638], [object_number=25, tag_tid=1, entry_tid=26637]]], state=connected] 2026-02-20T21:32:28.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:28.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:32:28.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-20T21:32:28.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:28.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:28.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:28.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-20T21:32:28.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:28.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:28.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:28.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:28.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-20T21:32:28.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-20T21:32:28.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:28.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-20T21:32:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:28.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:32:28.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:32:28.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:28.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638]]], state=connected] 2026-02-20T21:32:28.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26640], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638], [object_number=25, tag_tid=1, entry_tid=26637]]], state=connected] 2026-02-20T21:32:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:28.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=26640' '!=' 'object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:28.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:32:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_25 2026-02-20T21:32:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_25 ']' 2026-02-20T21:32:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-20T21:32:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-20T21:32:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:29.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-20T21:32:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:29.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:29.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:29.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:29.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-20T21:32:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-20T21:32:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-20T21:32:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-20T21:32:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-20T21:32:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-20T21:32:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:32:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:32:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638]]], state=connected] 2026-02-20T21:32:29.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26640], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638], [object_number=25, tag_tid=1, entry_tid=26637]]], state=connected] 2026-02-20T21:32:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:32:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-20T21:32:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-20T21:32:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-20T21:32:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-20T21:32:29.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-20T21:32:29.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:32:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:32:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638]]], state=connected] 2026-02-20T21:32:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26640], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638], [object_number=25, tag_tid=1, entry_tid=26637]]], state=connected] 2026-02-20T21:32:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=26640' '!=' 'object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:32:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_25 2026-02-20T21:32:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_25 ']' 2026-02-20T21:32:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-20T21:32:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-20T21:32:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:30.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:30.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:30.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-20T21:32:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:30.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:30.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:30.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:30.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:30.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-20T21:32:30.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-20T21:32:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-20T21:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-20T21:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-20T21:32:30.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:30.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:30.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-20T21:32:30.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:32:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:32:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638]]], state=connected] 2026-02-20T21:32:30.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26640], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638], [object_number=25, tag_tid=1, entry_tid=26637]]], state=connected] 2026-02-20T21:32:30.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:30.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:32:30.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-20T21:32:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-20T21:32:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-20T21:32:30.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-20T21:32:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-20T21:32:30.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:30.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:32:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:32:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638]]], state=connected] 2026-02-20T21:32:30.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=26640], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638], [object_number=25, tag_tid=1, entry_tid=26637]]], state=connected] 2026-02-20T21:32:30.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:30.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=26640' '!=' 'object_number=24, tag_tid=1, entry_tid=26640' 2026-02-20T21:32:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:30.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:32:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_25 2026-02-20T21:32:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:31.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_25 ']' 2026-02-20T21:32:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_25' 2026-02-20T21:32:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_25 2026-02-20T21:32:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:31.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_25 2026-02-20T21:32:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:31.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:31.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:31.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:31.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:31.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:31.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:31.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:31.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:31.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:31.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_25 2026-02-20T21:32:31.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:31.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_25 2026-02-20T21:32:31.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:31.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:31.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:31.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '' 2026-02-20T21:32:31.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:31.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:31.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:31.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:31.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-20T21:32:31.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-20T21:32:31.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:31.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:31.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-20T21:32:31.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:32:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:32:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638]]], state=connected] 2026-02-20T21:32:31.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638]]], state=connected] 2026-02-20T21:32:31.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:32:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_25 2026-02-20T21:32:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_25 '..*' 2026-02-20T21:32:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:32:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_25.status 2026-02-20T21:32:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_25.status 2026-02-20T21:32:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_25 2026-02-20T21:32:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:32:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:32:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638]]], state=connected] 2026-02-20T21:32:31.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26639], [object_number=26, tag_tid=1, entry_tid=26638]]], state=connected] 2026-02-20T21:32:31.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.status 2026-02-20T21:32:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:32:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:32:31.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_25 snap 2026-02-20T21:32:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:32:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:32:31.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:32:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_25@snap 2026-02-20T21:32:32.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_25': 2026-02-20T21:32:32.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:32:32.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:32:32.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:32:32.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 12229e38494d 2026-02-20T21:32:32.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.12229e38494d 2026-02-20T21:32:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:32:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:32:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:32:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:32:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:22:56 2026 2026-02-20T21:32:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:22:56 2026 2026-02-20T21:32:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:23:57 2026 2026-02-20T21:32:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:32:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 12229e38494d 2026-02-20T21:32:32.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:32:32.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:32:32.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 688c0a21-a152-4fd8-9e91-9df6e54166b4 2026-02-20T21:32:32.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:32:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_25 snap 2026-02-20T21:32:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:32:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:32:32.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.export 2026-02-20T21:32:32.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_25.export 2026-02-20T21:32:32.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_25.export 2026-02-20T21:32:32.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_25@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.export 2026-02-20T21:32:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:32:33.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_25@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_25.export 2026-02-20T21:32:33.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:32:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_25.export 2026-02-20T21:32:33.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_25.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_25.export 2026-02-20T21:32:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:32:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_26 2026-02-20T21:32:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_26 snap 2026-02-20T21:32:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_26@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:32:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_26 2026-02-20T21:32:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_26 started 2026-02-20T21:32:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:35.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:35.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:35.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:32:35.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:32:35.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:35.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:32:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_26 started 2026-02-20T21:32:36.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:32:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:32:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:32:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_26 2026-02-20T21:32:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:32:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:32:36.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:32:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:32:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:32:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:32:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_26 2026-02-20T21:32:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:32:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:32:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:32:36.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:36.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:36.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:36.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:36.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:36.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:36.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:36.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:32:36.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:32:36.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_26 2026-02-20T21:32:37.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:37.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:32:37.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_26 2026-02-20T21:32:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:32:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:37.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:32:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:32:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:32:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:32:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:37.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:37.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_26 2026-02-20T21:32:37.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:37.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:32:37.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:37.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:37.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_26 2026-02-20T21:32:37.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:32:37.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:37.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:37.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:32:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:32:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:32:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:32:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:37.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_26 2026-02-20T21:32:37.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:32:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:32:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:32:37.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:32:37.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:32:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:32:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:32:37.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_26 2026-02-20T21:32:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:32:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:32:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:32:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:32:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_26 2026-02-20T21:32:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:32:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:37.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:32:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:32:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:32:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:32:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:32:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:32:37.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-20T21:32:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-20T21:32:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-20T21:32:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-20T21:32:37.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-20T21:32:37.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:37.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:37.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:37.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:37.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:37.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:37.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:37.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-02-20T21:32:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:37.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:37.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:37.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:37.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:37.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-02-20T21:32:37.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:37.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:37.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:37.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:37.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:37.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-20T21:32:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:37.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-20T21:32:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-20T21:32:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-20T21:32:37.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:37.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:37.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-20T21:32:37.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:37.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:37.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:37.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:37.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:37.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45470], [object_number=41, tag_tid=1, entry_tid=45469], [object_number=40, tag_tid=1, entry_tid=45468], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:37.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-20T21:32:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-20T21:32:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-20T21:32:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-20T21:32:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:37.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-20T21:32:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:37.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45470], [object_number=41, tag_tid=1, entry_tid=45469], [object_number=40, tag_tid=1, entry_tid=45468], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:37.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=45470' '!=' '' 2026-02-20T21:32:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:32:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=45470' = '' 2026-02-20T21:32:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:32:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:37.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:32:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:32:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:32:37.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:37.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:32:37.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:37.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:32:37.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:32:37.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=45470 2026-02-20T21:32:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 45470 2026-02-20T21:32:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:32:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:37.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:32:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-20T21:32:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-20T21:32:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-20T21:32:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-20T21:32:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:37.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:37.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-20T21:32:37.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:37.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:37.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:37.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:37.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:37.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:37.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:37.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:37.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:37.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:37.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-02-20T21:32:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:37.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:37.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:37.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:37.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:37.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:37.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:37.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:37.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:37.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:37.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:37.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:37.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:37.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:37.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-02-20T21:32:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:37.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:37.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:37.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:37.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:37.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-20T21:32:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-20T21:32:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:37.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-20T21:32:37.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-20T21:32:37.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:37.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:37.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-20T21:32:37.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45470], [object_number=41, tag_tid=1, entry_tid=45469], [object_number=40, tag_tid=1, entry_tid=45468], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:37.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:37.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-20T21:32:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-20T21:32:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:38.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-20T21:32:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-20T21:32:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:38.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-20T21:32:38.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:38.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45470], [object_number=41, tag_tid=1, entry_tid=45469], [object_number=40, tag_tid=1, entry_tid=45468], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:38.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:38.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:38.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:38.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=45470' '!=' 'object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:38.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:38.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:32:38.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-20T21:32:38.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-20T21:32:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-20T21:32:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-20T21:32:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:38.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:38.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-20T21:32:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:38.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:38.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:38.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:38.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:38.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:38.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:38.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:38.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-02-20T21:32:38.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:38.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:38.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:38.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:38.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:38.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:38.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:38.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:38.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:38.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-02-20T21:32:38.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:38.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:38.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:38.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:38.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:38.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:38.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:38.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:38.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:38.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:38.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-20T21:32:38.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-20T21:32:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-20T21:32:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-20T21:32:38.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-20T21:32:38.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:38.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:38.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:38.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:38.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:38.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45470], [object_number=41, tag_tid=1, entry_tid=45469], [object_number=40, tag_tid=1, entry_tid=45468], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:38.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:38.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-20T21:32:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-20T21:32:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-20T21:32:38.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-20T21:32:38.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:38.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:38.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-20T21:32:38.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:38.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45470], [object_number=41, tag_tid=1, entry_tid=45469], [object_number=40, tag_tid=1, entry_tid=45468], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:38.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=45470' '!=' 'object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:38.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:39.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:39.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:39.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-20T21:32:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:39.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:39.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:39.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:39.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:39.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:39.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-02-20T21:32:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:39.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:39.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:39.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:39.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:39.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:39.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:39.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-02-20T21:32:39.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:39.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:39.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:39.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:39.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:39.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:39.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:39.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:39.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:39.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:39.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:39.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:39.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-20T21:32:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-20T21:32:39.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-20T21:32:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-20T21:32:39.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:39.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:39.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-20T21:32:39.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:39.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:39.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45470], [object_number=41, tag_tid=1, entry_tid=45469], [object_number=40, tag_tid=1, entry_tid=45468], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:39.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:39.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-20T21:32:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:39.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-20T21:32:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-20T21:32:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-20T21:32:39.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-20T21:32:39.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45470], [object_number=41, tag_tid=1, entry_tid=45469], [object_number=40, tag_tid=1, entry_tid=45468], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=45470' '!=' 'object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:32:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-20T21:32:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-20T21:32:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-20T21:32:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-20T21:32:41.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:41.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:41.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:41.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:41.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:41.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-02-20T21:32:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:41.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:41.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:41.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:41.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:41.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:41.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:41.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:41.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:41.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:41.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:41.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:41.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:41.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:41.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:41.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-20T21:32:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-20T21:32:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-20T21:32:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-20T21:32:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:41.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:41.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-20T21:32:41.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45470], [object_number=41, tag_tid=1, entry_tid=45469], [object_number=40, tag_tid=1, entry_tid=45468], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:41.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:41.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:41.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-20T21:32:41.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:41.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:41.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-20T21:32:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-20T21:32:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-20T21:32:41.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:41.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:41.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-20T21:32:41.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=42, tag_tid=1, entry_tid=45470], [object_number=41, tag_tid=1, entry_tid=45469], [object_number=40, tag_tid=1, entry_tid=45468], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:41.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=42, tag_tid=1, entry_tid=45470' '!=' 'object_number=42, tag_tid=1, entry_tid=45470' 2026-02-20T21:32:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_26 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_26 ']' 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_26' 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_26 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:43.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:43.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_26 2026-02-20T21:32:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:43.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:43.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:43.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:43.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:43.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_26 2026-02-20T21:32:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:43.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:43.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:43.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:43.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:43.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_26'] 2026-02-20T21:32:43.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:43.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_26 2026-02-20T21:32:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:43.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_26 2026-02-20T21:32:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_26 2026-02-20T21:32:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '' 2026-02-20T21:32:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-20T21:32:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-20T21:32:43.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:43.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:43.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-20T21:32:43.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:43.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_26 2026-02-20T21:32:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_26 '..*' 2026-02-20T21:32:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:32:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_26.status 2026-02-20T21:32:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_26.status 2026-02-20T21:32:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_26 2026-02-20T21:32:43.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:32:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:32:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=42, tag_tid=1, entry_tid=45470], [object_number=43, tag_tid=1, entry_tid=45467]]], state=connected] 2026-02-20T21:32:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.status 2026-02-20T21:32:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:32:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_26 snap 2026-02-20T21:32:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:32:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:32:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:32:44.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_26@snap 2026-02-20T21:32:44.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_26': 2026-02-20T21:32:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:32:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:32:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:32:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 12224d327f63 2026-02-20T21:32:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.12224d327f63 2026-02-20T21:32:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:32:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:32:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:32:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:32:44.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:23:04 2026 2026-02-20T21:32:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:23:04 2026 2026-02-20T21:32:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:26:23 2026 2026-02-20T21:32:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:32:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 12224d327f63 2026-02-20T21:32:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:32:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:32:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: f7cde03d-0247-46f6-87e3-fd748e24a495 2026-02-20T21:32:44.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:32:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_26 snap 2026-02-20T21:32:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:32:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:32:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.export 2026-02-20T21:32:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_26.export 2026-02-20T21:32:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_26.export 2026-02-20T21:32:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_26@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.export 2026-02-20T21:32:44.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:32:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_26@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_26.export 2026-02-20T21:32:45.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:32:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_26.export 2026-02-20T21:32:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_26.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_26.export 2026-02-20T21:32:45.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:45.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:32:45.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_27 2026-02-20T21:32:45.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_27 snap 2026-02-20T21:32:45.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:45.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:45.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:45.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:45.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_27@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:32:45.595 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.6272990437917887) > weight (0.25) 2026-02-20T21:32:45.595 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.42357387047654527) > weight (0.25) 2026-02-20T21:32:45.595 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.3024044742237527) > weight (0.25) 2026-02-20T21:32:45.595 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.3 2026-02-20T21:32:45.596 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Sent signal 15 2026-02-20T21:32:45.596 INFO:tasks.rbd_mirror_thrash:waiting for 3.083677281383561 secs before reviving daemons 2026-02-20T21:32:45.596 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:32:45.595+0000 7ff932213640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 109120) UID: 1000 2026-02-20T21:32:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_27 2026-02-20T21:32:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:46.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_27 started 2026-02-20T21:32:46.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:46.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:46.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:46.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:32:46.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:32:46.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:46.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:32:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_27 started 2026-02-20T21:32:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:47.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:32:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:32:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:32:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_27 2026-02-20T21:32:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:32:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:32:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:32:47.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:32:47.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:47.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:32:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:32:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_27 2026-02-20T21:32:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:32:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:47.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:47.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:32:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:32:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:47.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:47.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:32:47.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:32:47.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:47.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_27 2026-02-20T21:32:48.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-20T21:32:48.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:32:48.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_27 2026-02-20T21:32:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:32:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:32:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:32:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:48.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:48.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:48.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:48.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_27 2026-02-20T21:32:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:32:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:32:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:32:48.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:32:48.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:32:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:32:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:32:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_27 2026-02-20T21:32:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:32:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:32:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:32:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:32:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_27 2026-02-20T21:32:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:32:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:32:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:32:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:32:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:32:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:32:48.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:32:48.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_27 2026-02-20T21:32:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_27 ']' 2026-02-20T21:32:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-20T21:32:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-20T21:32:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:48.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:48.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:48.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:48.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:48.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:48.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:48.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:48.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-20T21:32:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:48.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:48.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:48.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:48.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:48.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:48.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:48.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:48.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:48.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:48.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:48.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-20T21:32:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-20T21:32:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-20T21:32:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:48.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:48.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:48.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:48.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:48.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-20T21:32:48.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:48.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-20T21:32:48.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:48.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:48.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:48.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-20T21:32:48.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:48.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:48.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-20T21:32:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-20T21:32:48.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-20T21:32:48.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:48.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=21204], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202], [object_number=17, tag_tid=1, entry_tid=21201]]], state=connected] 2026-02-20T21:32:48.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:48.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-20T21:32:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-20T21:32:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:48.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-20T21:32:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-20T21:32:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-20T21:32:48.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:48.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:48.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:48.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:48.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:48.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=21204], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202], [object_number=17, tag_tid=1, entry_tid=21201]]], state=connected] 2026-02-20T21:32:48.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=21204' '!=' '' 2026-02-20T21:32:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:32:48.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=21204' = '' 2026-02-20T21:32:48.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:48.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:48.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:32:48.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:48.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:32:48.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:48.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:32:48.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:48.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:32:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:32:48.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:48.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:32:48.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:48.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:32:48.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=21204 2026-02-20T21:32:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 21204 2026-02-20T21:32:48.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:32:48.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:48.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:32:48.681 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.3 2026-02-20T21:32:48.681 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:32:48.681 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.3 2026-02-20T21:32:48.681 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Restarting daemon 2026-02-20T21:32:48.681 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 2026-02-20T21:32:48.684 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Started 2026-02-20T21:32:48.684 INFO:tasks.rbd_mirror_thrash:waiting for 88.94678173630952 secs before thrashing 2026-02-20T21:32:48.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_27 2026-02-20T21:32:48.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_27 ']' 2026-02-20T21:32:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-20T21:32:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-20T21:32:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:48.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:48.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:48.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-20T21:32:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-20T21:32:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:48.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-20T21:32:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:48.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-20T21:32:48.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-20T21:32:48.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:48.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:48.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:48.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-20T21:32:48.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:48.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:48.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:48.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:48.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-20T21:32:48.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-20T21:32:48.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:48.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:48.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-20T21:32:48.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:48.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:48.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:48.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:48.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:48.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=21204], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202], [object_number=17, tag_tid=1, entry_tid=21201]]], state=connected] 2026-02-20T21:32:48.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:48.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:48.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-20T21:32:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-20T21:32:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-20T21:32:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-20T21:32:48.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:48.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:48.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-20T21:32:48.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=21204], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202], [object_number=17, tag_tid=1, entry_tid=21201]]], state=connected] 2026-02-20T21:32:48.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=21204' '!=' 'object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:32:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_27 2026-02-20T21:32:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_27 ']' 2026-02-20T21:32:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-20T21:32:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-20T21:32:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-20T21:32:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-20T21:32:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-20T21:32:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-20T21:32:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:49.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:49.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-20T21:32:49.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-20T21:32:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-20T21:32:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-20T21:32:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-20T21:32:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-20T21:32:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=21204], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202], [object_number=17, tag_tid=1, entry_tid=21201]]], state=connected] 2026-02-20T21:32:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-20T21:32:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-20T21:32:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:49.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-20T21:32:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-20T21:32:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-20T21:32:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:49.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:49.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:49.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:49.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:49.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=21204], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202], [object_number=17, tag_tid=1, entry_tid=21201]]], state=connected] 2026-02-20T21:32:49.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=21204' '!=' 'object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:49.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:49.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:32:50.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_27 2026-02-20T21:32:50.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:50.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:50.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:50.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:50.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_27 ']' 2026-02-20T21:32:50.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-20T21:32:50.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-20T21:32:50.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:50.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:50.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:50.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-20T21:32:50.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:50.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:50.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:50.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:50.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-20T21:32:50.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-20T21:32:50.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:50.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:50.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:50.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-20T21:32:50.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:50.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:50.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:50.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:50.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:50.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:50.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:50.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:50.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-20T21:32:50.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:50.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-20T21:32:50.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:50.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:50.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:50.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-20T21:32:50.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:50.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:50.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:50.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:50.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-20T21:32:50.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-20T21:32:50.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-20T21:32:50.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:50.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:50.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:50.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:50.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:50.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=21204], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202], [object_number=17, tag_tid=1, entry_tid=21201]]], state=connected] 2026-02-20T21:32:50.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:50.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-20T21:32:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-20T21:32:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:50.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-20T21:32:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-20T21:32:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:50.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-20T21:32:50.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:50.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:50.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:50.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:50.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:50.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=21204], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202], [object_number=17, tag_tid=1, entry_tid=21201]]], state=connected] 2026-02-20T21:32:50.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:50.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:50.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:50.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=21204' '!=' 'object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:50.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:50.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:32:52.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_27 2026-02-20T21:32:52.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:52.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:52.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_27 ']' 2026-02-20T21:32:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-20T21:32:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-20T21:32:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:52.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:52.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-20T21:32:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:52.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:52.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:52.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:52.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:52.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-20T21:32:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-20T21:32:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:52.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:52.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:52.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-20T21:32:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-20T21:32:52.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-20T21:32:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-20T21:32:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-20T21:32:52.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-20T21:32:52.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:52.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:52.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-20T21:32:52.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=21204], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202], [object_number=17, tag_tid=1, entry_tid=21201]]], state=connected] 2026-02-20T21:32:52.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:52.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-20T21:32:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:52.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-20T21:32:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-20T21:32:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-20T21:32:52.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-20T21:32:52.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=16, tag_tid=1, entry_tid=21204], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202], [object_number=17, tag_tid=1, entry_tid=21201]]], state=connected] 2026-02-20T21:32:52.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=16, tag_tid=1, entry_tid=21204' '!=' 'object_number=16, tag_tid=1, entry_tid=21204' 2026-02-20T21:32:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:32:54.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_27 2026-02-20T21:32:54.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:54.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:54.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:54.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:54.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_27 ']' 2026-02-20T21:32:54.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_27' 2026-02-20T21:32:54.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_27 2026-02-20T21:32:54.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:54.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:54.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:54.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:54.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:54.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:54.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:54.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:54.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_27 2026-02-20T21:32:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_27 2026-02-20T21:32:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_27'] 2026-02-20T21:32:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_27 2026-02-20T21:32:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:54.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:54.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:54.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:54.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:54.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:54.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_27 2026-02-20T21:32:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:54.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_27 2026-02-20T21:32:54.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:54.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '' 2026-02-20T21:32:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-20T21:32:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-20T21:32:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:54.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:54.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-20T21:32:54.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:54.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_27 2026-02-20T21:32:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_27 '..*' 2026-02-20T21:32:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:32:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_27.status 2026-02-20T21:32:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_27.status 2026-02-20T21:32:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_27 2026-02-20T21:32:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:54.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 4 2026-02-20T21:32:54.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 4 2026-02-20T21:32:54.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:54.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:54.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=17, tag_tid=2, entry_tid=1], [object_number=16, tag_tid=2, entry_tid=0], [object_number=19, tag_tid=1, entry_tid=21203], [object_number=18, tag_tid=1, entry_tid=21202]]], state=connected] 2026-02-20T21:32:54.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.status 2026-02-20T21:32:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=17, tag_tid=2, entry_tid=1' -a 'object_number=17, tag_tid=2, entry_tid=1' = 'object_number=17, tag_tid=2, entry_tid=1' 2026-02-20T21:32:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_27 snap 2026-02-20T21:32:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:32:54.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:32:54.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:32:55.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_27@snap 2026-02-20T21:32:55.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_27': 2026-02-20T21:32:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:32:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:32:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:32:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1212e835c4e0 2026-02-20T21:32:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1212e835c4e0 2026-02-20T21:32:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:32:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:32:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:32:55.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:32:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:23:18 2026 2026-02-20T21:32:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:23:18 2026 2026-02-20T21:32:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:24:19 2026 2026-02-20T21:32:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:32:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1212e835c4e0 2026-02-20T21:32:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:32:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:32:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: e59c41a5-f460-4ea0-8975-8b8a6f42ffa5 2026-02-20T21:32:55.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:32:55.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:55.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_27 snap 2026-02-20T21:32:55.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:55.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:32:55.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:55.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:32:55.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.export 2026-02-20T21:32:55.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_27.export 2026-02-20T21:32:55.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_27.export 2026-02-20T21:32:55.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_27@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.export 2026-02-20T21:32:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:32:55.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_27@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_27.export 2026-02-20T21:32:55.627 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:55.623+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:32:55.628 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:55.624+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:32:55.628 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:55.624+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:32:55.628 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:32:55.624+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:32:55.659 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:32:55.655+0000 7f4128cc4640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:32:55.659 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:32:55.655+0000 7f412ecd0640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:32:55.660 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:32:55.656+0000 7f412e4cf640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:32:55.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:32:55.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_27.export 2026-02-20T21:32:55.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_27.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_27.export 2026-02-20T21:32:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:32:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_28 2026-02-20T21:32:55.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_28 snap 2026-02-20T21:32:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:32:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:32:55.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_28@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:32:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_28 2026-02-20T21:32:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:57.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:32:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_28 started 2026-02-20T21:32:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:32:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:32:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:32:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:57.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:32:58.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_28 started 2026-02-20T21:32:58.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:58.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:58.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:32:58.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:32:58.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:32:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:32:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_28 2026-02-20T21:32:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:32:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:32:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:32:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:32:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:58.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:32:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:58.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:32:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_28 2026-02-20T21:32:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:32:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:58.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:32:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:32:58.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:32:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:32:58.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:58.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_28 2026-02-20T21:32:59.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T21:32:59.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:32:59.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_28 2026-02-20T21:32:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:32:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:32:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:32:59.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:59.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:59.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_28 2026-02-20T21:32:59.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T21:32:59.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:32:59.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:59.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:59.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_28 2026-02-20T21:32:59.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:32:59.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:32:59.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:59.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:59.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:59.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:32:59.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:32:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:32:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:32:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:32:59.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:32:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:32:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:32:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:32:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:32:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:32:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:32:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:32:59.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:59.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_28 2026-02-20T21:32:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:32:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:32:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:32:59.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:32:59.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:32:59.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:32:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:32:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_28 2026-02-20T21:32:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:32:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:32:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:32:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:32:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:32:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_28 2026-02-20T21:32:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:32:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:32:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:32:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:32:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:32:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:32:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:32:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:32:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:32:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_28 2026-02-20T21:32:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:32:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_28 ']' 2026-02-20T21:32:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-02-20T21:32:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-02-20T21:32:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:59.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:59.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:59.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:59.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:59.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:59.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:59.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-02-20T21:32:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:59.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:59.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:59.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:59.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:59.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:59.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:59.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:59.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:59.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:59.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:59.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:59.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:59.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:59.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-02-20T21:32:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T21:32:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:59.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-02-20T21:32:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:59.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:59.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:59.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:59.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:59.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:59.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:59.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:59.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:59.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:59.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-02-20T21:32:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T21:32:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_28 2026-02-20T21:32:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:59.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:59.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:59.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:59.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_28 2026-02-20T21:32:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-02-20T21:32:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:32:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-02-20T21:32:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:32:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-20T21:32:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-20T21:32:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:59.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:32:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-20T21:32:59.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:32:59.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:32:59.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:32:59.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:59.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370]]], state=connected] 2026-02-20T21:32:59.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370], [object_number=25, tag_tid=1, entry_tid=29369], [object_number=24, tag_tid=1, entry_tid=29368]]], state=connected] 2026-02-20T21:32:59.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:32:59.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:32:59.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-02-20T21:32:59.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:59.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:59.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:32:59.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-02-20T21:32:59.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:32:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:32:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-20T21:32:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-20T21:32:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:59.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:32:59.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-20T21:32:59.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:32:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:32:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:32:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:32:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370]]], state=connected] 2026-02-20T21:32:59.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370], [object_number=25, tag_tid=1, entry_tid=29369], [object_number=24, tag_tid=1, entry_tid=29368]]], state=connected] 2026-02-20T21:32:59.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:32:59.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=27, tag_tid=1, entry_tid=29371' 2026-02-20T21:32:59.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=27, tag_tid=1, entry_tid=29371' 2026-02-20T21:32:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=27, tag_tid=1, entry_tid=29371' '!=' '' 2026-02-20T21:32:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:32:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=27, tag_tid=1, entry_tid=29371' = '' 2026-02-20T21:32:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=27, tag_tid=1, entry_tid=29371' 2026-02-20T21:32:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:32:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:32:59.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:32:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=27, tag_tid=1, entry_tid=29371' 2026-02-20T21:32:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:32:59.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:59.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:32:59.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:32:59.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:32:59.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:59.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:32:59.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=27, tag_tid=1, entry_tid=29371' 2026-02-20T21:32:59.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:32:59.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:32:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29371 2026-02-20T21:32:59.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 29371 2026-02-20T21:32:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:32:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:32:59.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:32:59.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_28 2026-02-20T21:32:59.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:59.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:32:59.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:32:59.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:32:59.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_28 ']' 2026-02-20T21:32:59.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-02-20T21:32:59.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-02-20T21:32:59.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:32:59.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:59.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:32:59.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:32:59.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:32:59.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:32:59.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:32:59.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:32:59.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:32:59.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:59.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-02-20T21:32:59.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:32:59.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:59.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:59.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:32:59.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:32:59.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:59.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:59.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:59.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:32:59.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:59.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:59.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:32:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:32:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:32:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:32:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:32:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:32:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:32:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:32:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:32:59.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:32:59.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:32:59.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-02-20T21:32:59.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T21:32:59.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:59.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-02-20T21:32:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:32:59.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:32:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:32:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:32:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:32:59.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:59.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:32:59.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:32:59.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:59.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:59.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:32:59.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:32:59.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:59.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:59.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:32:59.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:32:59.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:59.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:59.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:32:59.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:32:59.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:32:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:32:59.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-02-20T21:32:59.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T21:32:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:32:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:32:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:32:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_28 2026-02-20T21:32:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:32:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:32:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:32:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:32:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:32:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:32:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:32:59.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:32:59.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:32:59.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:32:59.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:32:59.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:32:59.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:32:59.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:32:59.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:32:59.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:32:59.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:32:59.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:32:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:32:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_28 2026-02-20T21:32:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:32:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-02-20T21:32:59.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:32:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-02-20T21:32:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:32:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:32:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:32:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:32:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-20T21:32:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-20T21:32:59.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:32:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:32:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-20T21:32:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:33:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:33:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370]]], state=connected] 2026-02-20T21:33:00.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370], [object_number=25, tag_tid=1, entry_tid=29369], [object_number=24, tag_tid=1, entry_tid=29368]]], state=connected] 2026-02-20T21:33:00.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:00.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:33:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-02-20T21:33:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:33:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-02-20T21:33:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:33:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:00.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-20T21:33:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-20T21:33:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-20T21:33:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:33:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:33:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370]]], state=connected] 2026-02-20T21:33:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370], [object_number=25, tag_tid=1, entry_tid=29369], [object_number=24, tag_tid=1, entry_tid=29368]]], state=connected] 2026-02-20T21:33:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=27, tag_tid=1, entry_tid=29371' 2026-02-20T21:33:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=27, tag_tid=1, entry_tid=29371' 2026-02-20T21:33:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=27, tag_tid=1, entry_tid=29371' '!=' 'object_number=27, tag_tid=1, entry_tid=29371' 2026-02-20T21:33:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:33:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_28 2026-02-20T21:33:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:33:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:00.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_28 ']' 2026-02-20T21:33:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-02-20T21:33:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-02-20T21:33:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:00.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-02-20T21:33:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:00.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:00.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:00.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:00.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-02-20T21:33:00.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T21:33:00.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:00.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:00.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-02-20T21:33:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:00.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:00.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:00.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:00.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-02-20T21:33:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T21:33:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:00.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_28 2026-02-20T21:33:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:00.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:00.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_28 2026-02-20T21:33:00.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-02-20T21:33:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:00.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:33:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-02-20T21:33:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:33:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-20T21:33:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-20T21:33:00.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-20T21:33:00.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:00.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:33:00.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:33:00.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:00.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370]]], state=connected] 2026-02-20T21:33:00.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370], [object_number=25, tag_tid=1, entry_tid=29369], [object_number=24, tag_tid=1, entry_tid=29368]]], state=connected] 2026-02-20T21:33:00.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:00.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:33:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-02-20T21:33:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:33:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-02-20T21:33:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:00.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:33:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-20T21:33:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-20T21:33:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:00.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:00.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-20T21:33:00.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:00.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:33:00.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:33:00.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:00.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370]]], state=connected] 2026-02-20T21:33:00.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370], [object_number=25, tag_tid=1, entry_tid=29369], [object_number=24, tag_tid=1, entry_tid=29368]]], state=connected] 2026-02-20T21:33:00.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=27, tag_tid=1, entry_tid=29371' 2026-02-20T21:33:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=27, tag_tid=1, entry_tid=29371' 2026-02-20T21:33:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=27, tag_tid=1, entry_tid=29371' '!=' 'object_number=27, tag_tid=1, entry_tid=29371' 2026-02-20T21:33:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:33:01.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_28 2026-02-20T21:33:01.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:33:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_28 ']' 2026-02-20T21:33:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_28' 2026-02-20T21:33:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_28 2026-02-20T21:33:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:01.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_28 2026-02-20T21:33:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:01.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:01.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:01.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:01.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:01.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:01.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_28 2026-02-20T21:33:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T21:33:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:01.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:01.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:01.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_28 2026-02-20T21:33:01.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:01.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:01.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:01.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:01.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:01.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:01.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:01.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:01.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:01.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_28 2026-02-20T21:33:01.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_28'] 2026-02-20T21:33:01.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:01.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_28 2026-02-20T21:33:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:01.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:01.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:01.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:01.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:01.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:01.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:01.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:01.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:01.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:01.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:01.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:01.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:01.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:01.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:01.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:01.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_28 2026-02-20T21:33:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:01.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_28 2026-02-20T21:33:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:33:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '' 2026-02-20T21:33:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:33:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:01.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-20T21:33:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-20T21:33:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:01.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-20T21:33:01.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:33:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:33:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370]]], state=connected] 2026-02-20T21:33:01.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370]]], state=connected] 2026-02-20T21:33:01.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:33:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_28 2026-02-20T21:33:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:33:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_28 '..*' 2026-02-20T21:33:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:33:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_28.status 2026-02-20T21:33:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_28.status 2026-02-20T21:33:01.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_28 2026-02-20T21:33:01.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:33:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:33:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370]]], state=connected] 2026-02-20T21:33:01.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=29371], [object_number=26, tag_tid=1, entry_tid=29370]]], state=connected] 2026-02-20T21:33:01.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.status 2026-02-20T21:33:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:33:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:33:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_28 snap 2026-02-20T21:33:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:01.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:33:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:33:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:33:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:33:01.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:33:02.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_28@snap 2026-02-20T21:33:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_28': 2026-02-20T21:33:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:33:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:33:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:33:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 12227eda6e56 2026-02-20T21:33:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.12227eda6e56 2026-02-20T21:33:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:33:02.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:33:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:33:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:33:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:23:25 2026 2026-02-20T21:33:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:23:25 2026 2026-02-20T21:33:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:24:26 2026 2026-02-20T21:33:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:33:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 12227eda6e56 2026-02-20T21:33:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:33:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:33:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 8bdf974d-936c-4493-ad99-308729f14e8e 2026-02-20T21:33:02.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:33:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_28 snap 2026-02-20T21:33:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:33:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:33:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:33:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.export 2026-02-20T21:33:02.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_28.export 2026-02-20T21:33:02.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_28.export 2026-02-20T21:33:02.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_28@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.export 2026-02-20T21:33:02.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:33:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_28@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_28.export 2026-02-20T21:33:02.928 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:33:02.924+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:33:02.928 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:33:02.924+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:33:03.001 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:33:02.997+0000 7f412ecd0640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:33:03.003 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:33:02.999+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:33:03.003 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:33:02.999+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:33:03.003 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:33:02.999+0000 7f4128cc4640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:33:03.003 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:33:02.999+0000 7f4129cc6640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:33:03.004 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:33:03.000+0000 7f412fcd2640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 4 2026-02-20T21:33:03.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:33:03.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_28.export 2026-02-20T21:33:03.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_28.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_28.export 2026-02-20T21:33:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:33:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_29 2026-02-20T21:33:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_29 snap 2026-02-20T21:33:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:33:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:33:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_29@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:33:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_29 2026-02-20T21:33:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_29 started 2026-02-20T21:33:04.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:04.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:04.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:04.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:33:04.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:33:04.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:04.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:33:05.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_29 started 2026-02-20T21:33:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:33:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:33:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:33:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_29 2026-02-20T21:33:05.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:33:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:33:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:33:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:33:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:33:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:33:05.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:05.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_29 2026-02-20T21:33:05.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:33:05.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:33:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:33:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:33:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:33:05.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:05.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:05.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:05.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:05.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_29 2026-02-20T21:33:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_29 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:06.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:33:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:33:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:33:06.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:06.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_29 2026-02-20T21:33:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:06.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_29 2026-02-20T21:33:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:33:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:06.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:06.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:33:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:33:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:33:06.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:06.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:06.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_29 2026-02-20T21:33:06.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:33:06.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:33:06.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:33:06.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:33:06.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:33:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:33:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:33:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_29 2026-02-20T21:33:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:33:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:33:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:33:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_29 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:33:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_29 2026-02-20T21:33:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_29 ']' 2026-02-20T21:33:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-20T21:33:06.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-20T21:33:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:06.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-20T21:33:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:06.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:06.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:06.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:06.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:06.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:06.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:06.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:06.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:06.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-20T21:33:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:06.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:06.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:06.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:06.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:06.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:06.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:06.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:06.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:06.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:06.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-20T21:33:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:06.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:06.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:06.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:06.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:06.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:06.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-20T21:33:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-20T21:33:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-20T21:33:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-20T21:33:06.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-20T21:33:06.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:06.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34973], [object_number=32, tag_tid=1, entry_tid=34972], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:06.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:06.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-20T21:33:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-20T21:33:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:06.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-20T21:33:06.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-20T21:33:06.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:06.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:06.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-20T21:33:06.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:06.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:06.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34973], [object_number=32, tag_tid=1, entry_tid=34972], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:06.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34973' '!=' '' 2026-02-20T21:33:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:33:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34973' = '' 2026-02-20T21:33:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:33:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:33:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:33:06.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:06.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:33:06.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:33:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:33:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:33:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:33:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:33:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:33:06.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:33:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34973 2026-02-20T21:33:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 34973 2026-02-20T21:33:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:33:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:33:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_29 2026-02-20T21:33:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_29 ']' 2026-02-20T21:33:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-20T21:33:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-20T21:33:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:06.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:06.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:06.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:06.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:06.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:06.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:06.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:06.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-20T21:33:06.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:06.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:06.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:06.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:06.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:06.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:06.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:06.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:06.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:06.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:06.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:06.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:06.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:06.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:06.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:06.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:06.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:06.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:06.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:06.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:06.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-20T21:33:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:06.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:06.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:06.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:06.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:06.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:06.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:06.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:06.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:06.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-20T21:33:06.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:06.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:06.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:06.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:06.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:06.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:07.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:07.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-20T21:33:07.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:07.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:07.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-20T21:33:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-20T21:33:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-20T21:33:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:07.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:07.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-20T21:33:07.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:07.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34973], [object_number=32, tag_tid=1, entry_tid=34972], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:07.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:07.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-20T21:33:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-20T21:33:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-20T21:33:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-20T21:33:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:07.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:07.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-20T21:33:07.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:07.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:07.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34973], [object_number=32, tag_tid=1, entry_tid=34972], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:07.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:07.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:07.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34973' '!=' 'object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:07.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:33:07.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_29 2026-02-20T21:33:07.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:07.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:07.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:07.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:07.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_29 ']' 2026-02-20T21:33:07.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-20T21:33:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-20T21:33:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-20T21:33:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:07.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:07.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:07.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:07.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:07.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:07.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:07.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:07.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:07.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:07.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-20T21:33:07.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:07.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:07.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:07.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:07.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:07.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:07.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:07.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:07.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:07.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-20T21:33:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:07.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:07.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:07.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:07.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:07.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:07.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-20T21:33:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-20T21:33:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:07.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:07.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:07.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-20T21:33:07.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-20T21:33:07.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:07.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-20T21:33:07.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34973], [object_number=32, tag_tid=1, entry_tid=34972], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-20T21:33:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-20T21:33:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-20T21:33:07.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-20T21:33:07.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:07.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:07.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-20T21:33:07.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:07.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34973], [object_number=32, tag_tid=1, entry_tid=34972], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:07.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34973' '!=' 'object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:07.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:33:08.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_29 2026-02-20T21:33:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_29 ']' 2026-02-20T21:33:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-20T21:33:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-20T21:33:08.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:08.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-20T21:33:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:08.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:08.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:08.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:08.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:08.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:08.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:08.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-20T21:33:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:08.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:08.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:08.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:08.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:08.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:08.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:08.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:08.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-20T21:33:08.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:08.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:08.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:08.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:08.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:08.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:08.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:08.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:08.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:08.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-20T21:33:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:08.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-20T21:33:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-20T21:33:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-20T21:33:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-20T21:33:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34973], [object_number=32, tag_tid=1, entry_tid=34972], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:08.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:08.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-20T21:33:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-20T21:33:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-20T21:33:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-20T21:33:08.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:08.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:08.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-20T21:33:08.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:08.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:08.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34973], [object_number=32, tag_tid=1, entry_tid=34972], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:08.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:08.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34973' '!=' 'object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:08.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:33:10.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_29 2026-02-20T21:33:10.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:10.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_29 ']' 2026-02-20T21:33:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-20T21:33:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-20T21:33:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:10.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-20T21:33:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:10.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:10.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:10.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-20T21:33:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:10.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:10.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:10.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:10.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:10.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:10.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:10.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:10.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:10.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:10.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:10.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:10.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-20T21:33:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:10.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:10.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:10.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:10.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:10.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:10.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-20T21:33:10.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:10.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:10.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:10.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-20T21:33:10.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:10.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:10.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:10.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:10.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-20T21:33:10.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-20T21:33:10.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:10.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:10.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-20T21:33:10.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:10.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:10.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:10.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:10.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:10.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34973], [object_number=32, tag_tid=1, entry_tid=34972], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:10.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:10.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:10.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-20T21:33:10.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:10.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:10.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:10.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-20T21:33:10.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:10.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:10.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:10.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:10.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-20T21:33:10.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-20T21:33:10.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:10.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:10.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-20T21:33:10.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:10.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34973], [object_number=32, tag_tid=1, entry_tid=34972], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:10.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:10.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:10.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:10.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34973' '!=' 'object_number=33, tag_tid=1, entry_tid=34973' 2026-02-20T21:33:10.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:10.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_29 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_29 ']' 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_29' 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_29 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:12.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:12.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:12.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_29 2026-02-20T21:33:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:12.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:12.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:12.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:12.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:12.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:12.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:12.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:12.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:12.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:12.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:12.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:12.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_29 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:12.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:12.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:12.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:12.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:12.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:12.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:12.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:12.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:12.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:12.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:12.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_29'] 2026-02-20T21:33:12.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:12.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_29 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:12.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:12.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:12.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:12.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:12.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:12.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_29 2026-02-20T21:33:12.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_29 2026-02-20T21:33:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '' 2026-02-20T21:33:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-20T21:33:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-20T21:33:12.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:12.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:12.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-20T21:33:12.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:13.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:13.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_29 2026-02-20T21:33:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_29 '..*' 2026-02-20T21:33:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:33:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_29.status 2026-02-20T21:33:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_29.status 2026-02-20T21:33:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_29 2026-02-20T21:33:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:13.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34971], [object_number=34, tag_tid=1, entry_tid=34970]]], state=connected] 2026-02-20T21:33:13.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.status 2026-02-20T21:33:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_29 snap 2026-02-20T21:33:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:33:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:33:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:33:13.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:33:14.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_29@snap 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_29': 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ff7761df2f 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ff7761df2f 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:23:34 2026 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:23:34 2026 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:25:36 2026 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ff7761df2f 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 4353cd7f-67c5-4644-9ef9-e648f38ed9e8 2026-02-20T21:33:14.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:33:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_29 snap 2026-02-20T21:33:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:33:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:33:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:33:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.export 2026-02-20T21:33:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_29.export 2026-02-20T21:33:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_29.export 2026-02-20T21:33:14.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_29@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.export 2026-02-20T21:33:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:33:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_29@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_29.export 2026-02-20T21:33:14.277 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:33:14.274+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:33:14.277 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:33:14.274+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:33:14.277 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:33:14.274+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:33:14.278 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:33:14.275+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:33:14.278 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:33:14.275+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:33:14.278 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:33:14.275+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:33:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:33:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_29.export 2026-02-20T21:33:14.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_29.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_29.export 2026-02-20T21:33:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:33:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_30 2026-02-20T21:33:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_30 snap 2026-02-20T21:33:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:33:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:33:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_30@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:33:16.591 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.3902183974493887) > weight (0.25) 2026-02-20T21:33:16.592 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7472672663463023) > weight (0.25) 2026-02-20T21:33:16.592 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9888006366131019) > weight (0.25) 2026-02-20T21:33:16.592 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-20T21:33:16.592 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-20T21:33:16.592 INFO:tasks.rbd_mirror_thrash:waiting for 0.30698302358559526 secs before reviving daemons 2026-02-20T21:33:16.593 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:33:16.590+0000 7fc156730640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 97996) UID: 1000 2026-02-20T21:33:16.900 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-20T21:33:16.900 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:33:16.900 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-20T21:33:16.900 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-20T21:33:16.901 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-20T21:33:16.903 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-20T21:33:16.903 INFO:tasks.rbd_mirror_thrash:waiting for 75.54162397176191 secs before thrashing 2026-02-20T21:33:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_30 2026-02-20T21:33:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_30 started 2026-02-20T21:33:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:33:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:33:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:17.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:33:18.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_30 started 2026-02-20T21:33:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:33:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:33:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:33:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_30 2026-02-20T21:33:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:33:18.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:33:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:33:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:33:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:18.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:33:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:33:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-02-20T21:33:18.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:33:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:18.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:33:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:33:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:18.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:33:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:33:18.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:18.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-02-20T21:33:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:18.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:18.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2026-02-20T21:33:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:33:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:18.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:18.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:33:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:33:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:18.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:33:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:33:18.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2026-02-20T21:33:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:18.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:18.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:18.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2026-02-20T21:33:18.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:33:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:18.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:33:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:33:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:18.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:33:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:33:18.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:18.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2026-02-20T21:33:18.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:18.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:18.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_30 2026-02-20T21:33:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:33:18.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:18.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:18.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:33:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:33:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:18.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_30 2026-02-20T21:33:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:18.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:18.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 1 2026-02-20T21:33:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:19.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:33:19.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-02-20T21:33:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:33:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:19.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:33:19.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:33:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:19.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:33:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:33:19.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:19.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-02-20T21:33:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:19.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:19.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:19.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2026-02-20T21:33:19.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:33:19.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:19.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:33:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:33:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:19.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:33:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:33:19.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2026-02-20T21:33:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:19.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2026-02-20T21:33:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:33:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:19.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:19.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:33:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:33:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:19.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:33:19.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:33:19.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2026-02-20T21:33:19.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:19.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:19.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_30 2026-02-20T21:33:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:33:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:19.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:33:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:19.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:33:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:33:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:33:19.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_30 2026-02-20T21:33:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:19.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:19.512 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 2 2026-02-20T21:33:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:21.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:33:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-02-20T21:33:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:33:21.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:33:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:33:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:33:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:33:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:21.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:21.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:21.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:21.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-02-20T21:33:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:21.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2026-02-20T21:33:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:33:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:33:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:33:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:33:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:33:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:21.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:21.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:21.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:21.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2026-02-20T21:33:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:21.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2026-02-20T21:33:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:33:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:21.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:21.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:33:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2026-02-20T21:33:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_30 2026-02-20T21:33:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:33:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:21.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:33:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:21.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:33:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:33:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:33:21.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:21.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:21.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:21.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:21.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_30 2026-02-20T21:33:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 4 2026-02-20T21:33:25.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:25.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:33:25.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:25.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-02-20T21:33:25.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:33:25.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:33:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:33:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:33:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:33:25.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:25.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:25.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:25.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:25.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-02-20T21:33:25.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:25.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:25.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:25.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:25.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2026-02-20T21:33:25.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:33:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:33:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:33:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:33:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:33:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:25.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2026-02-20T21:33:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:25.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:25.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2026-02-20T21:33:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:33:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:25.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:33:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:33:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:25.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:33:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:33:25.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:25.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:25.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:25.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:25.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2026-02-20T21:33:25.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:25.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:25.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:25.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_30 2026-02-20T21:33:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:33:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:25.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:33:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:33:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:25.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:33:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:33:25.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:25.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_30 2026-02-20T21:33:25.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:25.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:25.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-20T21:33:33.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:33.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:33.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:33:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:33:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:33:33.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:34.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:34.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-02-20T21:33:34.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:34.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:34.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:34.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:33:34.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:34.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:34.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:34.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2026-02-20T21:33:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:34.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:33:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:34.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2026-02-20T21:33:34.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:34.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:34.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_30 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:34.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:33:34.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:34.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_30 2026-02-20T21:33:34.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:34.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:34.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:34.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:34.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-20T21:33:42.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:42.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:33:42.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:42.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-02-20T21:33:42.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:33:42.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:42.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:42.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:33:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:33:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:42.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:42.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:42.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:33:42.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:33:42.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:42.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-02-20T21:33:42.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2026-02-20T21:33:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:33:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:33:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:33:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:33:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:33:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:42.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:42.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:42.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:42.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2026-02-20T21:33:42.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:42.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:42.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:42.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:42.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2026-02-20T21:33:42.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:33:42.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:42.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:42.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:33:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:33:42.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:42.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:42.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:42.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:42.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2026-02-20T21:33:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_30 2026-02-20T21:33:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:33:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:42.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:33:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:33:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:42.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:33:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:33:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:42.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_30 2026-02-20T21:33:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-20T21:33:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:50.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_30 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:50.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:50.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:50.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:33:50.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:33:50.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:50.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:50.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:50.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:50.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_30 2026-02-20T21:33:50.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:50.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:50.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:50.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:50.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_30 2026-02-20T21:33:50.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:33:50.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:50.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:50.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:50.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:33:50.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:33:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:50.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:33:50.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:33:50.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_30 2026-02-20T21:33:50.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:50.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:50.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:50.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:50.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_30 2026-02-20T21:33:50.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:33:50.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:50.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:50.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:50.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:50.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:50.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:50.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:50.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:50.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:50.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:50.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:33:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:33:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:50.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:50.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:50.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:50.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_30 2026-02-20T21:33:50.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:50.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:33:50.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:50.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:50.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_30 2026-02-20T21:33:50.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:33:50.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:33:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:33:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:33:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:33:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:33:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:33:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:33:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:33:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:33:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:33:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:33:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:33:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:33:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:33:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:50.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:50.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:50.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:50.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_30 2026-02-20T21:33:50.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:33:50.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:33:50.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:33:50.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_30 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_30 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:33:50.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:33:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:33:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:33:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:33:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:33:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:50.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:33:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_30 2026-02-20T21:33:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:50.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_30 ']' 2026-02-20T21:33:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-20T21:33:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-20T21:33:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:50.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:50.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:50.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:50.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:50.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-20T21:33:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:50.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:50.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:50.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:50.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:50.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:50.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:50.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:50.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:50.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:50.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:50.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:50.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:50.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:50.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:50.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:50.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:50.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-20T21:33:50.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:50.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:50.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:50.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:50.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:50.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:50.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:50.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:50.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:50.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:50.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:50.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:50.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:51.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-20T21:33:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:51.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:51.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:51.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:51.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:51.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:51.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:51.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:51.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:51.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:51.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:51.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:51.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-20T21:33:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:33:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:33:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:33:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:33:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:33:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:51.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:51.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:51.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:51.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:51.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:51.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:51.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:51.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:51.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:51.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:51.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:51.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:51.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-20T21:33:51.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-20T21:33:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:33:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:33:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:33:51.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:51.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35877], [object_number=32, tag_tid=1, entry_tid=35876], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:51.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:51.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:51.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-20T21:33:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-20T21:33:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:33:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:33:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:33:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:51.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35877], [object_number=32, tag_tid=1, entry_tid=35876], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:51.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:51.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35877' '!=' '' 2026-02-20T21:33:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:33:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35877' = '' 2026-02-20T21:33:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:33:51.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:33:51.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:33:51.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:51.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:33:51.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:33:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:33:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:33:51.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:33:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:33:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:33:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:33:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=35877 2026-02-20T21:33:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 35877 2026-02-20T21:33:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:33:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:51.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:33:51.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_30 2026-02-20T21:33:51.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:51.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:51.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:51.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_30 ']' 2026-02-20T21:33:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-20T21:33:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-20T21:33:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:51.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-20T21:33:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:51.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:51.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:51.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:51.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:51.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:51.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:51.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:51.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:51.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:51.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-20T21:33:51.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:51.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:51.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:51.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:51.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:51.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:51.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:51.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:51.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-20T21:33:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:51.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:51.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:51.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:51.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:51.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:51.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:51.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:51.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:51.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-20T21:33:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:33:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:33:51.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:33:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:33:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:33:51.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:51.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:51.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:51.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:51.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:51.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:51.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-20T21:33:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-20T21:33:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:33:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:33:51.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:51.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:51.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:33:51.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35877], [object_number=32, tag_tid=1, entry_tid=35876], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-20T21:33:51.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-20T21:33:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:33:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:33:51.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:51.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:51.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:33:51.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:51.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:51.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:51.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:51.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:51.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35877], [object_number=32, tag_tid=1, entry_tid=35876], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:51.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:51.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35877' '!=' 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:33:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_30 2026-02-20T21:33:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_30 ']' 2026-02-20T21:33:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-20T21:33:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-20T21:33:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-20T21:33:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:52.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:52.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:52.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:52.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-20T21:33:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:52.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:52.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:52.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:52.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:52.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:52.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-20T21:33:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:52.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:52.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:52.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:52.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:52.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:52.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:52.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-20T21:33:52.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:33:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:33:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:52.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:33:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:33:52.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:33:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:52.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:52.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-20T21:33:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-20T21:33:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:33:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:33:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:33:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:52.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:52.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:52.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:52.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:52.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35877], [object_number=32, tag_tid=1, entry_tid=35876], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:52.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:52.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-20T21:33:52.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-20T21:33:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:33:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:33:52.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:52.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:52.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:33:52.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:52.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:52.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:52.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:52.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:52.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35877], [object_number=32, tag_tid=1, entry_tid=35876], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:52.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:52.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35877' '!=' 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:52.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:33:53.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_30 2026-02-20T21:33:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_30 ']' 2026-02-20T21:33:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-20T21:33:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-20T21:33:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-20T21:33:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:53.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:53.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:53.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-20T21:33:53.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:53.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:53.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:53.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:53.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:53.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:53.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:53.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:53.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-20T21:33:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:53.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:53.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:53.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:53.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:53.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:53.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-20T21:33:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:33:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:33:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:33:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:33:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:33:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-20T21:33:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-20T21:33:53.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:33:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:33:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:33:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35877], [object_number=32, tag_tid=1, entry_tid=35876], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:53.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:53.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-20T21:33:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-20T21:33:53.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:53.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:53.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:53.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:53.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:33:53.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:53.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:33:53.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:53.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:33:53.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35877], [object_number=32, tag_tid=1, entry_tid=35876], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:53.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:53.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:53.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:53.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35877' '!=' 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:53.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:53.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:33:55.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_30 2026-02-20T21:33:55.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_30 ']' 2026-02-20T21:33:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-20T21:33:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-20T21:33:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:55.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:55.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:55.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:55.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:55.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-20T21:33:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:55.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:55.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:55.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:55.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:55.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:55.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:55.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:55.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:55.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:55.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:55.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:55.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:55.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:55.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:55.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:55.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:55.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:55.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-20T21:33:55.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:55.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:55.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:55.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:55.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:55.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:55.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:55.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:55.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:55.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:55.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:55.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:55.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:55.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:55.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:55.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:55.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:55.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:55.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:55.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:55.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-20T21:33:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:55.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:55.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:55.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:55.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:55.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:55.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-20T21:33:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:33:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:33:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:55.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:33:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:33:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:33:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:55.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:55.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:55.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:55.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:55.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:55.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:55.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-20T21:33:55.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-20T21:33:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:33:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:33:55.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:33:55.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:55.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35877], [object_number=32, tag_tid=1, entry_tid=35876], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:55.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:55.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-20T21:33:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-20T21:33:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:55.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:33:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:33:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:55.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:33:55.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:55.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:55.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:55.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:55.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:55.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35877], [object_number=32, tag_tid=1, entry_tid=35876], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:55.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:55.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:55.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35877' '!=' 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:55.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:55.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:33:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_30 2026-02-20T21:33:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_30 ']' 2026-02-20T21:33:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-20T21:33:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-20T21:33:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-20T21:33:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:57.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:57.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:57.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:57.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:57.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:57.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:57.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:57.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:57.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:57.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:57.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-20T21:33:57.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:33:57.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:57.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:57.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:33:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:33:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:33:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:33:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:33:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:33:57.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:33:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:33:57.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:33:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:57.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:57.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:33:57.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:57.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-20T21:33:57.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:33:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:33:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:33:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:33:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:57.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:33:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:33:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:33:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:33:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:33:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:57.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:57.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:33:57.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:33:57.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:57.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:57.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:33:57.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:33:57.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:33:57.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:57.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:57.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:33:57.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:57.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:57.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:57.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:57.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:57.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-20T21:33:57.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:33:57.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:33:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:33:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:33:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:57.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:33:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:33:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:33:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:33:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:33:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:33:57.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:33:57.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:57.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:57.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:33:57.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:33:57.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:33:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:33:57.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:57.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:33:57.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-20T21:33:57.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-20T21:33:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:33:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:33:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:33:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:33:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:57.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:57.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:57.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:57.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:57.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35877], [object_number=32, tag_tid=1, entry_tid=35876], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:57.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:57.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:33:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-20T21:33:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-20T21:33:57.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:33:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:33:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:33:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:33:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:33:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:33:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:33:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:33:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:57.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:33:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:33:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:33:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:57.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=35877], [object_number=32, tag_tid=1, entry_tid=35876], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:33:57.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:33:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=35877' '!=' 'object_number=33, tag_tid=1, entry_tid=35877' 2026-02-20T21:33:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:33:57.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:33:59.355 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:33:59.353+0000 7f413d8ae640 -1 reset not still connected to 0x55d2752dd860 2026-02-20T21:33:59.356 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:33:59.353+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3040 2026-02-20T21:33:59.356 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:33:59.353+0000 7f413d8ae640 -1 reset not still connected to 0x55d2806b3e10 2026-02-20T21:33:59.356 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:33:59.353+0000 7f413d8ae640 -1 reset not still connected to 0x55d284334dd0 2026-02-20T21:33:59.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_30 2026-02-20T21:33:59.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:59.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:33:59.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:33:59.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:33:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_30 ']' 2026-02-20T21:33:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_30' 2026-02-20T21:33:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_30 2026-02-20T21:33:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:33:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:33:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:33:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:33:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:33:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:33:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:33:59.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:33:59.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:33:59.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_30 2026-02-20T21:33:59.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:33:59.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:33:59.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:33:59.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:33:59.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:33:59.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:33:59.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:33:59.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:33:59.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:33:59.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:33:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:33:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:33:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:33:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:33:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:33:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:33:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:33:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:33:59.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:33:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:33:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:33:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:33:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:33:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:33:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:33:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:33:59.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:33:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:33:59.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_30 2026-02-20T21:33:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:33:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:33:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:33:59.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_30 2026-02-20T21:34:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:00.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:00.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_30 2026-02-20T21:34:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:34:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_30 2026-02-20T21:34:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:34:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:34:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:34:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:34:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:34:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:00.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:00.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_30 2026-02-20T21:34:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_30'] 2026-02-20T21:34:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_30 2026-02-20T21:34:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:34:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:34:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:34:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:34:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:34:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:34:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:34:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:34:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:34:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:34:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:34:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:34:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:34:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:34:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:34:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:34:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:00.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:00.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:00.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_30 2026-02-20T21:34:00.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_30 2026-02-20T21:34:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:34:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '' 2026-02-20T21:34:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:34:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:34:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:34:00.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:34:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:34:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:34:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:34:00.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:34:00.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:34:00.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_30 2026-02-20T21:34:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:34:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_30 '..*' 2026-02-20T21:34:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:34:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_30.status 2026-02-20T21:34:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_30.status 2026-02-20T21:34:00.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:00.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:34:00.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_30 2026-02-20T21:34:00.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:34:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:34:00.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=35875], [object_number=34, tag_tid=1, entry_tid=35874]]], state=connected] 2026-02-20T21:34:00.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.status 2026-02-20T21:34:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_30 snap 2026-02-20T21:34:00.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:34:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:34:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:34:00.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:34:01.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_30@snap 2026-02-20T21:34:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_30': 2026-02-20T21:34:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:34:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:34:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:34:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11b3e97febe7 2026-02-20T21:34:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11b3e97febe7 2026-02-20T21:34:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:34:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:34:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:34:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:34:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:23:44 2026 2026-02-20T21:34:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:23:44 2026 2026-02-20T21:34:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:25:46 2026 2026-02-20T21:34:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:34:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11b3e97febe7 2026-02-20T21:34:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:34:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:34:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 224cb622-5e6a-4ad0-b67b-ff24ac29fe93 2026-02-20T21:34:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:34:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_30 snap 2026-02-20T21:34:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:34:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:34:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.export 2026-02-20T21:34:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_30.export 2026-02-20T21:34:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_30.export 2026-02-20T21:34:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_30@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.export 2026-02-20T21:34:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:34:01.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_30@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_30.export 2026-02-20T21:34:01.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:34:01.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_30.export 2026-02-20T21:34:01.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_30.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_30.export 2026-02-20T21:34:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:34:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_31 2026-02-20T21:34:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_31 snap 2026-02-20T21:34:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:01.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_31@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:34:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_31 2026-02-20T21:34:03.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_31 started 2026-02-20T21:34:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:34:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:34:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:03.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:34:04.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_31 started 2026-02-20T21:34:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:34:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:34:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:34:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_31 2026-02-20T21:34:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:34:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:34:04.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:34:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:34:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:04.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:34:04.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:34:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_31 2026-02-20T21:34:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:34:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:34:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:04.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:34:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:34:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:34:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:34:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:04.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:34:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:34:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:34:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:34:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:34:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:34:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:34:04.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_31 2026-02-20T21:34:04.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-20T21:34:04.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:34:04.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:04.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:04.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_31 2026-02-20T21:34:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:34:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:34:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:04.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:04.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:34:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:34:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:34:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:34:04.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:04.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:04.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:04.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:04.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_31 2026-02-20T21:34:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:34:04.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:34:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:34:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:34:04.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:34:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:34:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:34:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_31 2026-02-20T21:34:04.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:34:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:34:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:34:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:34:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_31 2026-02-20T21:34:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:34:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:34:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:34:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:04.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:34:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:34:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:34:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:04.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:34:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_31 2026-02-20T21:34:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:04.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_31 ']' 2026-02-20T21:34:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-20T21:34:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-20T21:34:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:04.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:04.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:04.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:04.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-20T21:34:04.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:04.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:04.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:04.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:04.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:04.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:04.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:04.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:04.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:04.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:04.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:04.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:04.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:04.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:04.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:04.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-20T21:34:04.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-20T21:34:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:04.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-20T21:34:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:04.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:04.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:04.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:04.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-20T21:34:04.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:04.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-20T21:34:04.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-20T21:34:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-20T21:34:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-20T21:34:04.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:04.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:04.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-20T21:34:04.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:04.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34977], [object_number=32, tag_tid=1, entry_tid=34976], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:04.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-20T21:34:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-20T21:34:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-20T21:34:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-20T21:34:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-20T21:34:04.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:04.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34977], [object_number=32, tag_tid=1, entry_tid=34976], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:04.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:04.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:04.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34977' '!=' '' 2026-02-20T21:34:04.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:34:04.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34977' = '' 2026-02-20T21:34:04.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:04.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:04.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:34:04.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:34:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:34:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:04.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:34:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:34:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:34:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:34:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=34977 2026-02-20T21:34:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 34977 2026-02-20T21:34:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:34:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:34:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_31 2026-02-20T21:34:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:04.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_31 ']' 2026-02-20T21:34:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-20T21:34:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-20T21:34:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:04.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:04.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:04.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-20T21:34:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:04.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:04.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:04.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:04.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:04.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:04.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:04.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-20T21:34:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-20T21:34:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:04.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-20T21:34:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:04.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:04.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:04.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:04.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:04.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:04.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:04.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-20T21:34:04.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:04.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-20T21:34:04.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:04.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:04.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-20T21:34:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-20T21:34:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-20T21:34:04.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-20T21:34:04.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:04.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34977], [object_number=32, tag_tid=1, entry_tid=34976], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:04.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:04.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-20T21:34:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-20T21:34:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:04.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-20T21:34:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-20T21:34:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:04.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-20T21:34:04.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:05.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:05.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34977], [object_number=32, tag_tid=1, entry_tid=34976], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:05.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:05.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34977' '!=' 'object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:05.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:34:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_31 2026-02-20T21:34:05.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:05.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:05.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:05.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:05.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_31 ']' 2026-02-20T21:34:05.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-20T21:34:05.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-20T21:34:05.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:05.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-20T21:34:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:05.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:05.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:05.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-20T21:34:05.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-20T21:34:05.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:05.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-20T21:34:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:05.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:05.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:05.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:05.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-20T21:34:05.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-20T21:34:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-20T21:34:05.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-20T21:34:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-20T21:34:05.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-20T21:34:05.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34977], [object_number=32, tag_tid=1, entry_tid=34976], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-20T21:34:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-20T21:34:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-20T21:34:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-20T21:34:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-20T21:34:05.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:05.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34977], [object_number=32, tag_tid=1, entry_tid=34976], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:05.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:05.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:05.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:05.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34977' '!=' 'object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:05.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:05.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:34:06.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_31 2026-02-20T21:34:06.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:06.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:06.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:06.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:06.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_31 ']' 2026-02-20T21:34:06.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-20T21:34:06.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-20T21:34:06.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:06.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:06.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:06.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:06.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:06.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:06.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-20T21:34:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:06.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:06.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:06.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:06.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:06.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:06.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:06.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:06.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:06.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:06.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:06.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-20T21:34:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-20T21:34:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-20T21:34:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:06.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:06.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:06.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:06.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-20T21:34:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-20T21:34:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-20T21:34:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-20T21:34:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-20T21:34:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:06.676 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-20T21:34:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:06.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:06.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34977], [object_number=32, tag_tid=1, entry_tid=34976], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:06.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:06.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:06.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-20T21:34:06.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:06.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:06.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:06.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-20T21:34:06.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:06.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:06.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-20T21:34:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-20T21:34:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:06.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:06.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-20T21:34:06.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:06.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:06.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:06.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:06.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:06.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34977], [object_number=32, tag_tid=1, entry_tid=34976], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:06.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34977' '!=' 'object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:06.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:34:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_31 2026-02-20T21:34:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_31 ']' 2026-02-20T21:34:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-20T21:34:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-20T21:34:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-20T21:34:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-20T21:34:08.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-20T21:34:08.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:08.605 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:08.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-20T21:34:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:08.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-20T21:34:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-20T21:34:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:08.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-20T21:34:08.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:08.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:08.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:08.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:08.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-20T21:34:08.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-20T21:34:08.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:08.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:08.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-20T21:34:08.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:08.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:08.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:08.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:08.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:08.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34977], [object_number=32, tag_tid=1, entry_tid=34976], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:08.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:08.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-20T21:34:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-20T21:34:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-20T21:34:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-20T21:34:08.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:08.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:08.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-20T21:34:08.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:08.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:08.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=1, entry_tid=34977], [object_number=32, tag_tid=1, entry_tid=34976], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:08.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=33, tag_tid=1, entry_tid=34977' '!=' 'object_number=33, tag_tid=1, entry_tid=34977' 2026-02-20T21:34:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:08.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:34:10.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_31 2026-02-20T21:34:10.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_31 ']' 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_31' 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_31 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:10.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:10.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_31 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:10.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:10.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:10.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:10.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:10.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_31 2026-02-20T21:34:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_31'] 2026-02-20T21:34:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:10.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_31 2026-02-20T21:34:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:10.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:10.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:10.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:10.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_31 2026-02-20T21:34:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_31 2026-02-20T21:34:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '' 2026-02-20T21:34:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:10.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:10.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:10.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:10.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-20T21:34:10.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-20T21:34:10.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:10.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:10.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-20T21:34:10.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:10.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:10.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:10.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:10.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_31 2026-02-20T21:34:10.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:10.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:10.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:10.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_31 '..*' 2026-02-20T21:34:10.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:10.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:10.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:34:10.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:10.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_31.status 2026-02-20T21:34:10.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_31.status 2026-02-20T21:34:10.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_31 2026-02-20T21:34:10.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=34975], [object_number=34, tag_tid=1, entry_tid=34974]]], state=connected] 2026-02-20T21:34:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.status 2026-02-20T21:34:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_31 snap 2026-02-20T21:34:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:34:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:34:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:34:11.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_31@snap 2026-02-20T21:34:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_31': 2026-02-20T21:34:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:34:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:34:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:34:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 12129868121 2026-02-20T21:34:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.12129868121 2026-02-20T21:34:11.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:34:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:34:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:34:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:34:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:23:53 2026 2026-02-20T21:34:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:23:53 2026 2026-02-20T21:34:11.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:25:55 2026 2026-02-20T21:34:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:34:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 12129868121 2026-02-20T21:34:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:34:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:34:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: d8e9deb3-b09b-4cb1-b33c-60d1177d3e4e 2026-02-20T21:34:11.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:34:11.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_31 snap 2026-02-20T21:34:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:34:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:34:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.export 2026-02-20T21:34:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_31.export 2026-02-20T21:34:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_31.export 2026-02-20T21:34:11.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_31@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.export 2026-02-20T21:34:12.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:34:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_31@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_31.export 2026-02-20T21:34:12.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:34:12.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_31.export 2026-02-20T21:34:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_31.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_31.export 2026-02-20T21:34:12.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:34:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_32 2026-02-20T21:34:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_32 snap 2026-02-20T21:34:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:12.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_32@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:34:14.399 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:34:14.398+0000 7f03c8435640 -1 reset not still connected to 0x55afb1e4e5b0 2026-02-20T21:34:14.400 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:34:14.398+0000 7f03c8435640 -1 reset not still connected to 0x55afc5039a00 2026-02-20T21:34:14.400 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:34:14.398+0000 7f03c8435640 -1 reset not still connected to 0x55afd0b06d00 2026-02-20T21:34:14.400 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:34:14.398+0000 7f03c8435640 -1 reset not still connected to 0x55afd468f5f0 2026-02-20T21:34:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_32 2026-02-20T21:34:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_32 started 2026-02-20T21:34:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:34:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:34:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:15.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:34:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_32 started 2026-02-20T21:34:16.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:34:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:34:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:34:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_32 2026-02-20T21:34:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:34:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:34:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:34:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:34:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:34:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:34:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_32 2026-02-20T21:34:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:34:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:34:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:34:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:16.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:34:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:34:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:34:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:34:16.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:34:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:34:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:34:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:34:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:34:16.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:34:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:16.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:16.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:16.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:16.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_32 2026-02-20T21:34:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:34:16.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_32 2026-02-20T21:34:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:34:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:34:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:34:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:34:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:34:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:34:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:34:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:34:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:34:16.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:16.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_32 2026-02-20T21:34:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:34:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:16.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:16.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_32 2026-02-20T21:34:16.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:34:16.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:16.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:16.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:34:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:34:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:34:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:34:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:34:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:34:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:34:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:34:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:34:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:34:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:34:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:34:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:16.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:16.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_32 2026-02-20T21:34:16.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:34:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:34:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:34:16.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:34:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:34:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:34:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:34:16.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_32 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_32 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:16.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:34:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_32 2026-02-20T21:34:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_32 ']' 2026-02-20T21:34:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-20T21:34:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-20T21:34:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:16.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:16.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-20T21:34:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:16.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:16.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:16.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:16.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:16.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-20T21:34:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:16.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:16.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:16.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:16.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:16.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:16.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:16.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-02-20T21:34:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:34:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:34:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:34:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:34:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:34:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:16.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:16.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:16.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-20T21:34:16.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:16.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:16.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:16.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-20T21:34:16.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:16.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:16.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:16.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:16.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:16.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:16.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:16.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:16.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:16.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:16.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=27558], [object_number=25, tag_tid=1, entry_tid=27557], [object_number=24, tag_tid=1, entry_tid=27556], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:16.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:16.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:16.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-20T21:34:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-20T21:34:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:16.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:16.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:16.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:16.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:16.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:16.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=27558], [object_number=25, tag_tid=1, entry_tid=27557], [object_number=24, tag_tid=1, entry_tid=27556], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:16.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:16.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:16.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=27558' '!=' '' 2026-02-20T21:34:16.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:34:16.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=27558' = '' 2026-02-20T21:34:16.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:16.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:34:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:16.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:34:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:34:16.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:34:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:16.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:34:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:34:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:34:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:16.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27558 2026-02-20T21:34:16.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 27558 2026-02-20T21:34:16.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:34:16.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:16.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:34:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_32 2026-02-20T21:34:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_32 ']' 2026-02-20T21:34:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-20T21:34:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-20T21:34:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-20T21:34:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:16.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:16.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:16.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:16.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:16.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:16.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:16.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:16.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:16.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-20T21:34:16.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:16.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:16.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:16.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:16.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:16.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:16.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:16.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:17.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-02-20T21:34:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:34:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:34:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:34:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:34:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:34:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:17.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:17.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:17.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:17.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:17.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-20T21:34:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-20T21:34:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:17.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:17.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:17.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:17.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:17.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=27558], [object_number=25, tag_tid=1, entry_tid=27557], [object_number=24, tag_tid=1, entry_tid=27556], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:17.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:17.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-20T21:34:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-20T21:34:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:17.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:17.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:17.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:17.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=27558], [object_number=25, tag_tid=1, entry_tid=27557], [object_number=24, tag_tid=1, entry_tid=27556], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=27558' '!=' 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_32 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_32 ']' 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:17.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:17.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:17.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:17.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:17.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:17.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:17.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:17.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:17.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:17.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:17.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:17.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:17.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:17.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:17.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:17.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:17.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:17.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:17.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:17.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:17.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:17.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:17.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-20T21:34:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:17.632 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-20T21:34:17.632 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-20T21:34:17.633 INFO:tasks.rbd_mirror_thrash:waiting for 0.09372568843178719 secs before reviving daemons 2026-02-20T21:34:17.633 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:34:17.631+0000 7f719e297640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 104340) UID: 1000 2026-02-20T21:34:17.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:17.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:17.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:17.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:17.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:17.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:17.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-02-20T21:34:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:34:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:34:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:34:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:34:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:34:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:17.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:17.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:17.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:17.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:17.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:17.727 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-20T21:34:17.727 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:34:17.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:17.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-20T21:34:17.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:17.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-20T21:34:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:17.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:17.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:17.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:17.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:17.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:17.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:17.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=27558], [object_number=25, tag_tid=1, entry_tid=27557], [object_number=24, tag_tid=1, entry_tid=27556], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:17.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:17.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:17.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-20T21:34:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-20T21:34:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:17.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:17.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:17.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=27558], [object_number=25, tag_tid=1, entry_tid=27557], [object_number=24, tag_tid=1, entry_tid=27556], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:17.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:17.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:17.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:17.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=27558' '!=' 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:17.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:17.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:34:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_32 2026-02-20T21:34:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_32 ']' 2026-02-20T21:34:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-20T21:34:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-20T21:34:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:18.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:18.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-20T21:34:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:18.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:18.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:18.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:18.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:18.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:18.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:18.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:18.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:18.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-20T21:34:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:18.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:18.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:18.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:18.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:18.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:18.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:18.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:18.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-02-20T21:34:18.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:34:18.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:18.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:18.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:18.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:34:18.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:18.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:18.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:18.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:34:18.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:18.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:18.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:18.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:34:18.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:18.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:18.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:34:18.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:18.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:18.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:18.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-20T21:34:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:18.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-20T21:34:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:18.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:18.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=27558], [object_number=25, tag_tid=1, entry_tid=27557], [object_number=24, tag_tid=1, entry_tid=27556], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:18.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-20T21:34:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-20T21:34:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:18.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:18.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:18.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:18.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:18.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:18.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:18.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:18.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=27558], [object_number=25, tag_tid=1, entry_tid=27557], [object_number=24, tag_tid=1, entry_tid=27556], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=27558' '!=' 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:34:20.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_32 2026-02-20T21:34:20.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:20.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:20.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_32 ']' 2026-02-20T21:34:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-20T21:34:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-20T21:34:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:20.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-20T21:34:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:20.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:20.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:20.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:20.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:20.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:20.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:20.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:20.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:20.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:20.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:20.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:20.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:20.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:20.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:20.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:20.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:20.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:20.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:20.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:20.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:20.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:20.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:20.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:20.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-02-20T21:34:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:34:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:34:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:20.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:20.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:20.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:20.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:20.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:20.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:20.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:20.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-20T21:34:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-20T21:34:20.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:20.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:20.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:20.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:20.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:20.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:20.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:20.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:20.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=27558], [object_number=25, tag_tid=1, entry_tid=27557], [object_number=24, tag_tid=1, entry_tid=27556], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:20.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:20.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-20T21:34:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-20T21:34:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:20.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:20.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:20.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:20.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:20.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:20.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:20.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=27558], [object_number=25, tag_tid=1, entry_tid=27557], [object_number=24, tag_tid=1, entry_tid=27556], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:20.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:20.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=27558' '!=' 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:20.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:34:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_32 2026-02-20T21:34:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:22.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_32 ']' 2026-02-20T21:34:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-20T21:34:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-20T21:34:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:22.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:22.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-20T21:34:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:22.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:22.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:22.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:22.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:22.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:22.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:22.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:22.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:22.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:22.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:22.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:22.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:22.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-20T21:34:22.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:22.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:22.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:22.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:22.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-02-20T21:34:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:34:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:34:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:22.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:22.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:22.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:34:22.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:22.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:22.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:22.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:34:22.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:22.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:22.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:34:22.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:22.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:22.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:22.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:22.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:22.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:22.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:22.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:22.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:22.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:22.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:22.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:22.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:22.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:22.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:22.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:22.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:22.851 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:22.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-20T21:34:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-20T21:34:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:22.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:22.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:22.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:22.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=27558], [object_number=25, tag_tid=1, entry_tid=27557], [object_number=24, tag_tid=1, entry_tid=27556], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:22.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-20T21:34:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-20T21:34:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:22.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:22.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:22.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:22.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:22.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:22.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:22.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=27558], [object_number=25, tag_tid=1, entry_tid=27557], [object_number=24, tag_tid=1, entry_tid=27556], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:22.962 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:22.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=27558' '!=' 'object_number=26, tag_tid=1, entry_tid=27558' 2026-02-20T21:34:22.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:22.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:34:23.828 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-20T21:34:23.828 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T21:34:23.829 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T21:34:23.831 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T21:34:23.831 INFO:tasks.rbd_mirror_thrash:waiting for 62.01253668203667 secs before thrashing 2026-02-20T21:34:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_32 2026-02-20T21:34:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_32 ']' 2026-02-20T21:34:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_32' 2026-02-20T21:34:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_32 2026-02-20T21:34:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_32 2026-02-20T21:34:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:24.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:25.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_32 2026-02-20T21:34:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:25.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:25.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:25.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:25.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:25.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:25.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:25.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_32'] 2026-02-20T21:34:25.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:25.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_32 2026-02-20T21:34:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:34:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:25.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:34:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:34:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:34:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:34:25.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:25.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:25.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:25.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:25.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:25.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_32 2026-02-20T21:34:25.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_32 2026-02-20T21:34:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '' 2026-02-20T21:34:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:25.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:25.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:25.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:25.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:25.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_32 2026-02-20T21:34:25.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:25.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:25.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:25.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_32 '..*' 2026-02-20T21:34:25.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:25.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:25.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:34:25.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:25.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_32.status 2026-02-20T21:34:25.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_32.status 2026-02-20T21:34:25.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:25.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:25.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_32 2026-02-20T21:34:25.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:25.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=27558], [object_number=27, tag_tid=1, entry_tid=27555]]], state=connected] 2026-02-20T21:34:25.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.status 2026-02-20T21:34:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_32 snap 2026-02-20T21:34:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:34:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:34:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:34:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_32@snap 2026-02-20T21:34:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_32': 2026-02-20T21:34:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:34:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:34:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:34:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 122273a418bd 2026-02-20T21:34:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.122273a418bd 2026-02-20T21:34:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:34:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:34:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:34:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:34:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:24:03 2026 2026-02-20T21:34:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:24:03 2026 2026-02-20T21:34:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:26:24 2026 2026-02-20T21:34:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:34:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 122273a418bd 2026-02-20T21:34:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:34:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:34:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: b77544d4-9eac-484b-bae9-bb8d0b03b1c0 2026-02-20T21:34:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:34:26.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_32 snap 2026-02-20T21:34:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:34:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:34:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.export 2026-02-20T21:34:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_32.export 2026-02-20T21:34:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_32.export 2026-02-20T21:34:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_32@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.export 2026-02-20T21:34:26.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:34:26.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_32@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_32.export 2026-02-20T21:34:26.402 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:26.399+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:34:26.402 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:26.399+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:34:26.403 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:26.400+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:34:26.404 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:26.400+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:34:26.404 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:26.400+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:34:26.404 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:26.401+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:34:26.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:34:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_32.export 2026-02-20T21:34:26.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_32.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_32.export 2026-02-20T21:34:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:34:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_33 2026-02-20T21:34:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_33 snap 2026-02-20T21:34:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:34:26.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:26.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_33@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:34:28.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_33 2026-02-20T21:34:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:34:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_33 started 2026-02-20T21:34:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:34:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:34:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:34:28.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:28.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:34:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_33 started 2026-02-20T21:34:29.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_33 2026-02-20T21:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:34:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:34:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_33 2026-02-20T21:34:29.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:34:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:34:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:34:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:29.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:34:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:34:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:34:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:34:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:34:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:34:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:29.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:34:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:34:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:34:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:34:29.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:29.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:29.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:29.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:29.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_33 2026-02-20T21:34:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:34:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_33 2026-02-20T21:34:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:34:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:34:29.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:29.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:34:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:34:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:34:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:34:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:34:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:34:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:34:29.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:29.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:29.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_33 2026-02-20T21:34:29.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:29.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:34:29.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_33 2026-02-20T21:34:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:34:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:34:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:34:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:34:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:34:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:34:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:34:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:34:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:34:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:34:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:34:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:34:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:34:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:29.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_33 2026-02-20T21:34:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:34:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_33 2026-02-20T21:34:29.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:34:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:34:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:34:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:34:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:34:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:34:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:34:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:29.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:34:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:34:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:34:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:34:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:34:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:34:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:34:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:34:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:34:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:34:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:29.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:34:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:34:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:34:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:34:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:34:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:34:29.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:34:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_33 2026-02-20T21:34:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:34:29.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:34:29.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:34:29.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:34:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:34:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:34:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:34:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_33 2026-02-20T21:34:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:34:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:34:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:34:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:34:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:34:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_33 2026-02-20T21:34:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:34:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:34:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:34:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:34:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:34:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:34:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:34:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:34:29.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_33 2026-02-20T21:34:29.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:29.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:29.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:34:29.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:29.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_33 ']' 2026-02-20T21:34:29.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-20T21:34:29.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-20T21:34:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:29.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:29.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-20T21:34:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:29.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:29.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:29.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:29.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:29.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:29.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:29.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:29.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:29.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-20T21:34:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:29.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:29.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:29.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:29.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:29.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:29.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:29.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:29.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:29.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:29.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:29.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:29.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:29.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:29.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:29.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:29.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:29.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:29.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:29.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-20T21:34:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:34:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:34:29.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:29.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:29.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:29.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:29.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:29.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:29.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:29.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_33 2026-02-20T21:34:29.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:34:29.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:29.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:34:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:34:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:34:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:34:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:34:29.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:29.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:29.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:34:29.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:34:29.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:34:29.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:29.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:29.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:29.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:29.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-20T21:34:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:29.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-20T21:34:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-20T21:34:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-20T21:34:29.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-20T21:34:29.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838]]], state=connected] 2026-02-20T21:34:29.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=36840], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838], [object_number=33, tag_tid=1, entry_tid=36837]]], state=connected] 2026-02-20T21:34:29.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:29.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:29.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-20T21:34:29.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:29.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:29.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:29.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-20T21:34:29.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:29.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:29.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:29.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:29.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-20T21:34:29.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-20T21:34:29.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:29.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:29.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-20T21:34:29.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:29.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838]]], state=connected] 2026-02-20T21:34:29.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=36840], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838], [object_number=33, tag_tid=1, entry_tid=36837]]], state=connected] 2026-02-20T21:34:29.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:29.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=36840' 2026-02-20T21:34:29.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=36840' 2026-02-20T21:34:29.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=36840' '!=' '' 2026-02-20T21:34:29.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:34:29.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=36840' = '' 2026-02-20T21:34:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=32, tag_tid=1, entry_tid=36840' 2026-02-20T21:34:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:34:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:29.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:34:29.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=32, tag_tid=1, entry_tid=36840' 2026-02-20T21:34:29.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:34:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:29.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:34:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:34:29.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:29.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:34:29.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=32, tag_tid=1, entry_tid=36840' 2026-02-20T21:34:29.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:34:29.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:29.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=36840 2026-02-20T21:34:29.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 36840 2026-02-20T21:34:29.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:34:29.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:29.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:34:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_33 2026-02-20T21:34:30.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:34:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_33 ']' 2026-02-20T21:34:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-20T21:34:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-20T21:34:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:30.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:30.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:30.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:30.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:30.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-20T21:34:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:30.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:30.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:30.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:30.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:30.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:30.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-20T21:34:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:30.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:30.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:30.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:30.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:30.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:30.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:30.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:30.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:30.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-20T21:34:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:34:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:34:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:34:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:34:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:30.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:30.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:30.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:30.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:30.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:30.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:30.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:30.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:30.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_33 2026-02-20T21:34:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:34:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:34:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:34:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:34:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:34:30.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:34:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:34:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:34:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:34:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:34:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:34:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:30.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:34:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:34:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:34:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:34:30.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:34:30.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:30.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:30.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:30.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:30.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-20T21:34:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-20T21:34:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-20T21:34:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-20T21:34:30.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:30.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:30.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-20T21:34:30.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:30.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838]]], state=connected] 2026-02-20T21:34:30.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=36840], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838], [object_number=33, tag_tid=1, entry_tid=36837]]], state=connected] 2026-02-20T21:34:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-20T21:34:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-20T21:34:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-20T21:34:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-20T21:34:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-20T21:34:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:30.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838]]], state=connected] 2026-02-20T21:34:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=36840], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838], [object_number=33, tag_tid=1, entry_tid=36837]]], state=connected] 2026-02-20T21:34:30.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=36840' 2026-02-20T21:34:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=36840' 2026-02-20T21:34:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=36840' '!=' 'object_number=32, tag_tid=1, entry_tid=36840' 2026-02-20T21:34:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_33 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_33 ']' 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:30.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:30.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:30.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-20T21:34:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:30.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:30.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:30.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:30.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-20T21:34:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:34:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:34:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:34:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:34:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:34:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:30.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:30.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:30.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:30.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_33 2026-02-20T21:34:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:34:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:34:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:34:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:34:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:34:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:34:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:34:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:34:30.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:34:30.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:30.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:30.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:30.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-20T21:34:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-20T21:34:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-20T21:34:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-20T21:34:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-20T21:34:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:31.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:31.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:31.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:31.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838]]], state=connected] 2026-02-20T21:34:31.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=36840], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838], [object_number=33, tag_tid=1, entry_tid=36837]]], state=connected] 2026-02-20T21:34:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:31.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:31.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-20T21:34:31.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:31.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-20T21:34:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-20T21:34:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-20T21:34:31.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-20T21:34:31.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838]]], state=connected] 2026-02-20T21:34:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=32, tag_tid=1, entry_tid=36840], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838], [object_number=33, tag_tid=1, entry_tid=36837]]], state=connected] 2026-02-20T21:34:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=32, tag_tid=1, entry_tid=36840' 2026-02-20T21:34:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=32, tag_tid=1, entry_tid=36840' 2026-02-20T21:34:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=32, tag_tid=1, entry_tid=36840' '!=' 'object_number=32, tag_tid=1, entry_tid=36840' 2026-02-20T21:34:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:31.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_33 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_33 ']' 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_33' 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_33 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_33 2026-02-20T21:34:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:31.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:31.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_33 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:34:31.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:34:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:34:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:34:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_33 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:34:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:34:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:34:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:32.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_33'] 2026-02-20T21:34:32.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:34:32.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_33 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:34:32.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:32.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:32.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:34:32.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:34:32.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:34:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:34:32.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_33 2026-02-20T21:34:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_33 2026-02-20T21:34:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:32.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '' 2026-02-20T21:34:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-20T21:34:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-20T21:34:32.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:32.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:32.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-20T21:34:32.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:32.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:32.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:32.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:32.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838]]], state=connected] 2026-02-20T21:34:32.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838]]], state=connected] 2026-02-20T21:34:32.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:32.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_33 2026-02-20T21:34:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_33 '..*' 2026-02-20T21:34:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:32.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:34:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_33.status 2026-02-20T21:34:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_33.status 2026-02-20T21:34:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_33 2026-02-20T21:34:32.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 8 2026-02-20T21:34:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 8 2026-02-20T21:34:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838]]], state=connected] 2026-02-20T21:34:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=33, tag_tid=2, entry_tid=1], [object_number=32, tag_tid=2, entry_tid=0], [object_number=35, tag_tid=1, entry_tid=36839], [object_number=34, tag_tid=1, entry_tid=36838]]], state=connected] 2026-02-20T21:34:32.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.status 2026-02-20T21:34:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=33, tag_tid=2, entry_tid=1' -a 'object_number=33, tag_tid=2, entry_tid=1' = 'object_number=33, tag_tid=2, entry_tid=1' 2026-02-20T21:34:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_33 snap 2026-02-20T21:34:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:34:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:34:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:34:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:34:32.446 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.47406614755498266) > weight (0.25) 2026-02-20T21:34:32.446 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.7530293239151367) > weight (0.25) 2026-02-20T21:34:32.446 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.8932836701182466) > weight (0.25) 2026-02-20T21:34:32.446 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.7070471042021322) > weight (0.25) 2026-02-20T21:34:32.446 INFO:tasks.rbd_mirror_thrash:waiting for 101.43326341581624 secs before thrashing 2026-02-20T21:34:33.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_33@snap 2026-02-20T21:34:33.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_33': 2026-02-20T21:34:33.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:34:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:34:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:34:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ffc06ad36b 2026-02-20T21:34:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ffc06ad36b 2026-02-20T21:34:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:34:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:34:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:34:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:34:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:24:11 2026 2026-02-20T21:34:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:24:11 2026 2026-02-20T21:34:33.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:26:13 2026 2026-02-20T21:34:33.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:34:33.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ffc06ad36b 2026-02-20T21:34:33.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:34:33.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:34:33.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 2f30e07e-4de9-4add-a743-a6c8365a5ecc 2026-02-20T21:34:33.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:34:33.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:33.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_33 snap 2026-02-20T21:34:33.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:33.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:34:33.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:33.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:34:33.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.export 2026-02-20T21:34:33.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_33.export 2026-02-20T21:34:33.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_33.export 2026-02-20T21:34:33.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_33@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.export 2026-02-20T21:34:33.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:34:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_33@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_33.export 2026-02-20T21:34:33.410 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:34:33.407+0000 7fc9be008640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2026-02-20T21:34:33.411 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:34:33.407+0000 7fc9c2811640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 2 2026-02-20T21:34:33.411 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:34:33.407+0000 7fc9c3012640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 2 2026-02-20T21:34:33.412 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:34:33.408+0000 7fc9c000c640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 2 2026-02-20T21:34:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:34:33.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_33.export 2026-02-20T21:34:33.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_33.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_33.export 2026-02-20T21:34:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:34:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_34 2026-02-20T21:34:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_34 snap 2026-02-20T21:34:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:33.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_34@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:34:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_34 2026-02-20T21:34:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:36.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_34 started 2026-02-20T21:34:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:34:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:34:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:36.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:34:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_34 started 2026-02-20T21:34:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:34:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:34:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:34:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_34 2026-02-20T21:34:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:34:37.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:37.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:34:37.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:37.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:34:37.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:34:37.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:37.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:34:37.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:37.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:34:37.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:37.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_34 2026-02-20T21:34:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:34:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:34:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:34:37.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:34:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:34:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:34:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:34:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:34:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:34:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:37.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:34:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:34:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:34:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:34:37.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:37.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:37.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:37.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:37.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_34 2026-02-20T21:34:37.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:34:37.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:34:37.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:34:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:34:37.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:34:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:34:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:34:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_34 2026-02-20T21:34:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:34:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:37.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:34:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:34:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:34:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_34 2026-02-20T21:34:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:34:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:34:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:34:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:34:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:34:37.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:34:37.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:37.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:34:37.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_34 2026-02-20T21:34:37.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:37.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_34 ']' 2026-02-20T21:34:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-20T21:34:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-20T21:34:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:37.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:37.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:37.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:37.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-20T21:34:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:37.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:37.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:37.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-20T21:34:37.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-20T21:34:37.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-20T21:34:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:37.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:37.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:37.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:37.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26174], [object_number=25, tag_tid=1, entry_tid=26173], [object_number=24, tag_tid=1, entry_tid=26172], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:37.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:37.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-20T21:34:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-20T21:34:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:37.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26174], [object_number=25, tag_tid=1, entry_tid=26173], [object_number=24, tag_tid=1, entry_tid=26172], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:37.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:37.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26174' '!=' '' 2026-02-20T21:34:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:34:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26174' = '' 2026-02-20T21:34:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:34:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:37.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:34:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:34:37.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:34:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:34:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:37.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:34:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:34:37.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26174 2026-02-20T21:34:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 26174 2026-02-20T21:34:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:34:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:37.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:34:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_34 2026-02-20T21:34:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:37.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_34 ']' 2026-02-20T21:34:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-20T21:34:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-20T21:34:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:37.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:37.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:37.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:37.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:37.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-20T21:34:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:37.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:37.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:37.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:37.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:37.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:37.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:37.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-20T21:34:37.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-20T21:34:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-20T21:34:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:37.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:37.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:37.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26174], [object_number=25, tag_tid=1, entry_tid=26173], [object_number=24, tag_tid=1, entry_tid=26172], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:37.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:37.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-20T21:34:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:37.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-20T21:34:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:37.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:37.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:37.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:37.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:37.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:37.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:37.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:37.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:37.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26174], [object_number=25, tag_tid=1, entry_tid=26173], [object_number=24, tag_tid=1, entry_tid=26172], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:37.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:37.829 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:37.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26174' '!=' 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:37.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:37.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:34:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_34 2026-02-20T21:34:38.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_34 ']' 2026-02-20T21:34:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-20T21:34:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-20T21:34:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:38.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:38.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:38.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:38.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-20T21:34:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:38.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:38.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:38.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:38.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:38.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:38.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:38.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-20T21:34:38.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-20T21:34:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-20T21:34:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:38.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:38.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26174], [object_number=25, tag_tid=1, entry_tid=26173], [object_number=24, tag_tid=1, entry_tid=26172], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:38.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:38.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-20T21:34:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-20T21:34:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:38.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:38.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:38.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:38.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:38.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:38.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:38.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26174], [object_number=25, tag_tid=1, entry_tid=26173], [object_number=24, tag_tid=1, entry_tid=26172], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:38.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:38.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:38.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26174' '!=' 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:38.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:38.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:34:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_34 2026-02-20T21:34:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:39.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_34 ']' 2026-02-20T21:34:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-20T21:34:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-20T21:34:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:39.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:39.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-20T21:34:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:39.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:39.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:39.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:39.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:39.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:39.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:39.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-20T21:34:39.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-20T21:34:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-20T21:34:39.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:39.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:39.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:39.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26174], [object_number=25, tag_tid=1, entry_tid=26173], [object_number=24, tag_tid=1, entry_tid=26172], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-20T21:34:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-20T21:34:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:39.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:39.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:39.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:39.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:39.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:39.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:39.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:39.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26174], [object_number=25, tag_tid=1, entry_tid=26173], [object_number=24, tag_tid=1, entry_tid=26172], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:39.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26174' '!=' 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:39.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:34:40.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_34 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_34 ']' 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:40.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-20T21:34:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:40.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:40.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-20T21:34:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-20T21:34:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-20T21:34:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:40.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:40.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:40.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:40.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:40.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26174], [object_number=25, tag_tid=1, entry_tid=26173], [object_number=24, tag_tid=1, entry_tid=26172], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:40.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-20T21:34:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-20T21:34:40.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:40.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:40.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:40.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:40.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:41.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26174], [object_number=25, tag_tid=1, entry_tid=26173], [object_number=24, tag_tid=1, entry_tid=26172], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:41.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26174' '!=' 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:41.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:34:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_34 2026-02-20T21:34:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:43.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_34 ']' 2026-02-20T21:34:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-20T21:34:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-20T21:34:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:43.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:43.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:43.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-20T21:34:43.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:43.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:43.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:43.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:43.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:43.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:43.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:43.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:43.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:43.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:43.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:43.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:43.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-20T21:34:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:43.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-20T21:34:43.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:43.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-20T21:34:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:43.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26174], [object_number=25, tag_tid=1, entry_tid=26173], [object_number=24, tag_tid=1, entry_tid=26172], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:43.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:43.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-20T21:34:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:43.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-20T21:34:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:43.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:43.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:43.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:43.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:43.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:43.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=26174], [object_number=25, tag_tid=1, entry_tid=26173], [object_number=24, tag_tid=1, entry_tid=26172], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:43.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=26174' '!=' 'object_number=26, tag_tid=1, entry_tid=26174' 2026-02-20T21:34:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:43.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:34:45.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_34 2026-02-20T21:34:45.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:45.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:45.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_34 ']' 2026-02-20T21:34:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_34' 2026-02-20T21:34:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_34 2026-02-20T21:34:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:45.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:45.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_34 2026-02-20T21:34:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:45.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:45.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:45.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:45.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:45.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_34 2026-02-20T21:34:45.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_34 2026-02-20T21:34:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '' 2026-02-20T21:34:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:45.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:45.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:45.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:45.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:45.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:45.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:45.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_34 2026-02-20T21:34:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_34 '..*' 2026-02-20T21:34:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:34:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_34.status 2026-02-20T21:34:45.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_34.status 2026-02-20T21:34:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_34 2026-02-20T21:34:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:45.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:34:45.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:34:45.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:45.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:45.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=26174], [object_number=27, tag_tid=1, entry_tid=26171]]], state=connected] 2026-02-20T21:34:45.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.status 2026-02-20T21:34:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:34:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:45.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_34 snap 2026-02-20T21:34:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:34:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:34:45.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:34:46.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_34@snap 2026-02-20T21:34:46.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_34': 2026-02-20T21:34:46.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:34:46.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:34:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:34:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11b3daea269 2026-02-20T21:34:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11b3daea269 2026-02-20T21:34:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:34:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:34:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:34:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:34:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:24:21 2026 2026-02-20T21:34:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:24:21 2026 2026-02-20T21:34:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:25:22 2026 2026-02-20T21:34:46.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:34:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11b3daea269 2026-02-20T21:34:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:34:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:34:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 2698178d-280a-4f1d-818e-d530eb5de75b 2026-02-20T21:34:46.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:34:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_34 snap 2026-02-20T21:34:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:46.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:34:46.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:46.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:34:46.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.export 2026-02-20T21:34:46.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_34.export 2026-02-20T21:34:46.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_34.export 2026-02-20T21:34:46.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_34@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.export 2026-02-20T21:34:46.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:34:46.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_34@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_34.export 2026-02-20T21:34:46.525 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:46.521+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:34:46.525 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:46.522+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:34:46.526 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:46.522+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:34:46.526 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:46.522+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:34:46.527 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:46.524+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:34:46.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:34:46.604 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_34.export 2026-02-20T21:34:46.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_34.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_34.export 2026-02-20T21:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_35 2026-02-20T21:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_35 snap 2026-02-20T21:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:46.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_35@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:34:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_35 2026-02-20T21:34:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_35 started 2026-02-20T21:34:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:34:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:34:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_35 started 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_35 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:48.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:34:48.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:48.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_35 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:34:48.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:34:48.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_35 2026-02-20T21:34:49.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:34:49.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:34:49.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:34:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:34:49.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_35 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_35 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:34:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:34:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:34:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-20T21:34:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:49.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-20T21:34:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-20T21:34:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-20T21:34:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-20T21:34:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:49.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:49.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-20T21:34:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-20T21:34:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-20T21:34:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:49.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:49.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24937], [object_number=20, tag_tid=1, entry_tid=24936], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:34:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-20T21:34:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-20T21:34:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:49.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24937], [object_number=20, tag_tid=1, entry_tid=24936], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:49.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24937' '!=' '' 2026-02-20T21:34:49.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:34:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24937' = '' 2026-02-20T21:34:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:34:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:34:49.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:34:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:34:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:34:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:34:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:34:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:34:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:34:49.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:34:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24937 2026-02-20T21:34:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 24937 2026-02-20T21:34:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:34:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:49.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:49.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:49.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:49.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:49.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:49.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-20T21:34:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-20T21:34:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-20T21:34:49.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:49.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:49.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:49.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:49.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:49.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:49.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:49.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:49.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:49.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:49.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24937], [object_number=20, tag_tid=1, entry_tid=24936], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:49.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:49.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:34:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-20T21:34:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-20T21:34:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:49.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:49.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:49.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:49.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24937], [object_number=20, tag_tid=1, entry_tid=24936], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:49.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24937' '!=' 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:49.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:50.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:50.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:50.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:50.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-20T21:34:50.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:50.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-20T21:34:50.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:50.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:50.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:50.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-20T21:34:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:50.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:50.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:50.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:50.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24937], [object_number=20, tag_tid=1, entry_tid=24936], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:50.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:34:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-20T21:34:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-20T21:34:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:50.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:50.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:50.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:50.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24937], [object_number=20, tag_tid=1, entry_tid=24936], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:50.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:50.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24937' '!=' 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:50.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:34:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-20T21:34:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:51.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-20T21:34:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-20T21:34:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-20T21:34:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:51.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-20T21:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:51.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:51.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:51.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:51.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:51.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-20T21:34:51.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-20T21:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-20T21:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:51.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:51.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:51.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:51.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:51.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:51.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:51.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:51.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:51.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:51.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24937], [object_number=20, tag_tid=1, entry_tid=24936], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:51.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:51.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:34:51.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-20T21:34:51.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-20T21:34:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:51.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:51.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:51.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:51.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:51.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:51.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:51.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24937], [object_number=20, tag_tid=1, entry_tid=24936], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:51.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:51.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:51.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:51.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24937' '!=' 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:51.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:51.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:34:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-20T21:34:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-20T21:34:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-20T21:34:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-20T21:34:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-20T21:34:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-20T21:34:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-20T21:34:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:53.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:53.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:53.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:53.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:53.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24937], [object_number=20, tag_tid=1, entry_tid=24936], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:53.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:53.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:34:53.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-20T21:34:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-20T21:34:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:53.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:53.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:53.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:53.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:53.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24937], [object_number=20, tag_tid=1, entry_tid=24936], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:53.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:53.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:53.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:53.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24937' '!=' 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:53.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:53.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:34:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-20T21:34:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:55.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-20T21:34:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-20T21:34:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-20T21:34:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:55.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:55.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:55.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-20T21:34:55.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:55.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:55.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:55.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:55.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-20T21:34:55.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-20T21:34:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-20T21:34:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:55.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:55.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:55.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:55.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24937], [object_number=20, tag_tid=1, entry_tid=24936], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:55.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:55.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:34:55.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-20T21:34:55.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:55.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:55.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:55.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-20T21:34:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:55.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:55.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:55.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:55.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:55.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:55.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:55.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24937], [object_number=20, tag_tid=1, entry_tid=24936], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:55.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24937' '!=' 'object_number=21, tag_tid=1, entry_tid=24937' 2026-02-20T21:34:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:34:55.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:34:57.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_35 2026-02-20T21:34:57.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:57.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:57.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:57.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:34:57.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_35 ']' 2026-02-20T21:34:57.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_35' 2026-02-20T21:34:57.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_35 2026-02-20T21:34:57.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:34:57.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:57.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:57.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:34:57.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:34:57.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:34:57.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:34:57.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:34:57.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:34:57.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:34:57.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_35 2026-02-20T21:34:57.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:34:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:34:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:34:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:34:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:34:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:34:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:34:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:34:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:34:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:34:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:34:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:57.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:34:57.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:34:57.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:34:57.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:34:57.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:34:57.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:34:57.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:34:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:34:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:34:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:34:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:34:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:34:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:34:57.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:34:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:34:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:34:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:34:57.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_35 2026-02-20T21:34:57.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_35 2026-02-20T21:34:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '' 2026-02-20T21:34:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:34:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:57.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:57.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:57.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:57.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:57.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:57.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:57.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:34:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_35 2026-02-20T21:34:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_35 '..*' 2026-02-20T21:34:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:34:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:34:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:34:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:34:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_35.status 2026-02-20T21:34:57.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_35.status 2026-02-20T21:34:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:34:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_35 2026-02-20T21:34:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:57.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:34:57.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:34:57.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:34:57.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24935], [object_number=22, tag_tid=1, entry_tid=24934]]], state=connected] 2026-02-20T21:34:57.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.status 2026-02-20T21:34:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:34:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:34:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_35 snap 2026-02-20T21:34:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:34:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:34:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:34:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:34:58.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_35@snap 2026-02-20T21:34:58.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_35': 2026-02-20T21:34:58.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:34:58.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:34:58.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:34:58.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1212d9e53fb7 2026-02-20T21:34:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1212d9e53fb7 2026-02-20T21:34:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:34:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:34:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:34:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:34:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:24:28 2026 2026-02-20T21:34:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:24:28 2026 2026-02-20T21:34:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:25:29 2026 2026-02-20T21:34:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:34:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1212d9e53fb7 2026-02-20T21:34:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:34:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:34:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 0b809041-0acd-4fce-872a-bd142f193f3c 2026-02-20T21:34:58.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:34:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_35 snap 2026-02-20T21:34:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:58.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:34:58.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:58.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:34:58.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.export 2026-02-20T21:34:58.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_35.export 2026-02-20T21:34:58.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_35.export 2026-02-20T21:34:58.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_35@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.export 2026-02-20T21:34:58.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:34:58.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_35@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_35.export 2026-02-20T21:34:58.543 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:58.540+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:34:58.544 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:34:58.540+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:34:58.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:34:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_35.export 2026-02-20T21:34:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_35.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_35.export 2026-02-20T21:34:58.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:34:58.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:34:58.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_36 2026-02-20T21:34:58.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_36 snap 2026-02-20T21:34:58.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:34:58.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:34:58.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:34:58.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:34:58.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_36@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:35:00.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_36 2026-02-20T21:35:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_36 started 2026-02-20T21:35:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:35:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:35:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:00.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:35:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_36 started 2026-02-20T21:35:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:35:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:35:01.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:35:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_36 2026-02-20T21:35:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:35:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:35:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:35:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:35:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:01.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:35:01.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:01.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:35:01.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_36 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:01.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:35:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:35:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:35:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:35:01.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:01.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_36 2026-02-20T21:35:02.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:35:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:35:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:35:02.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:35:02.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_36 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_36 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:35:02.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_36 2026-02-20T21:35:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_36 ']' 2026-02-20T21:35:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_36' 2026-02-20T21:35:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_36 2026-02-20T21:35:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_36 2026-02-20T21:35:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:02.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:02.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:02.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:02.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:02.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_36 2026-02-20T21:35:02.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_36 2026-02-20T21:35:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:02.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '' 2026-02-20T21:35:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:02.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:02.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:02.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:02.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:02.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=29962], [object_number=29, tag_tid=1, entry_tid=29961], [object_number=28, tag_tid=1, entry_tid=29960], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:02.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:02.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:35:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_36 2026-02-20T21:35:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '..*' 2026-02-20T21:35:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:02.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=29962], [object_number=29, tag_tid=1, entry_tid=29961], [object_number=28, tag_tid=1, entry_tid=29960], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=29962' '!=' '' 2026-02-20T21:35:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:35:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=29962' = '' 2026-02-20T21:35:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:35:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:35:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:35:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:35:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:35:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:35:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:35:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:35:02.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:02.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:35:02.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29962 2026-02-20T21:35:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 29962 2026-02-20T21:35:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:35:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:35:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_36 2026-02-20T21:35:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_36 ']' 2026-02-20T21:35:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_36' 2026-02-20T21:35:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_36 2026-02-20T21:35:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:02.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:02.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:02.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_36 2026-02-20T21:35:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:02.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:02.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:02.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:02.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:02.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:02.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:02.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_36 2026-02-20T21:35:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_36 2026-02-20T21:35:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '' 2026-02-20T21:35:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:02.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:02.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:02.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:02.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:02.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:02.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:02.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:02.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:02.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=29962], [object_number=29, tag_tid=1, entry_tid=29961], [object_number=28, tag_tid=1, entry_tid=29960], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:02.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:02.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:35:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_36 2026-02-20T21:35:02.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '..*' 2026-02-20T21:35:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:02.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:02.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:02.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:02.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:02.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:02.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:02.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=29962], [object_number=29, tag_tid=1, entry_tid=29961], [object_number=28, tag_tid=1, entry_tid=29960], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:02.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=29962' '!=' 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:02.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:35:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_36 2026-02-20T21:35:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_36 ']' 2026-02-20T21:35:03.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_36' 2026-02-20T21:35:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_36 2026-02-20T21:35:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:03.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_36 2026-02-20T21:35:03.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:03.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:03.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:03.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_36 2026-02-20T21:35:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_36 2026-02-20T21:35:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '' 2026-02-20T21:35:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:03.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:03.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:03.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=29962], [object_number=29, tag_tid=1, entry_tid=29961], [object_number=28, tag_tid=1, entry_tid=29960], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:03.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:03.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:35:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_36 2026-02-20T21:35:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:03.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '..*' 2026-02-20T21:35:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:03.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:03.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=29962], [object_number=29, tag_tid=1, entry_tid=29961], [object_number=28, tag_tid=1, entry_tid=29960], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=29962' '!=' 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:35:04.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_36 2026-02-20T21:35:04.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:04.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:04.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_36 ']' 2026-02-20T21:35:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_36' 2026-02-20T21:35:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_36 2026-02-20T21:35:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:04.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:04.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:04.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:04.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_36 2026-02-20T21:35:04.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:04.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:04.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:04.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:04.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:04.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:04.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_36 2026-02-20T21:35:04.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:04.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_36 2026-02-20T21:35:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '' 2026-02-20T21:35:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:04.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:04.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:04.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:04.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:04.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:04.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:04.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=29962], [object_number=29, tag_tid=1, entry_tid=29961], [object_number=28, tag_tid=1, entry_tid=29960], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:04.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:04.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:35:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_36 2026-02-20T21:35:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '..*' 2026-02-20T21:35:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:04.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:04.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:04.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=29962], [object_number=29, tag_tid=1, entry_tid=29961], [object_number=28, tag_tid=1, entry_tid=29960], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:04.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:04.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:04.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:04.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=29962' '!=' 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:04.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:04.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:35:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_36 2026-02-20T21:35:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_36 ']' 2026-02-20T21:35:05.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_36' 2026-02-20T21:35:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_36 2026-02-20T21:35:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_36 2026-02-20T21:35:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:05.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:05.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:05.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_36 2026-02-20T21:35:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_36 2026-02-20T21:35:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '' 2026-02-20T21:35:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:05.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:05.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=29962], [object_number=29, tag_tid=1, entry_tid=29961], [object_number=28, tag_tid=1, entry_tid=29960], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:35:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_36 2026-02-20T21:35:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '..*' 2026-02-20T21:35:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:05.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:05.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=29962], [object_number=29, tag_tid=1, entry_tid=29961], [object_number=28, tag_tid=1, entry_tid=29960], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:05.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:05.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=29962' '!=' 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:35:07.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_36 2026-02-20T21:35:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_36 ']' 2026-02-20T21:35:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_36' 2026-02-20T21:35:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_36 2026-02-20T21:35:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:07.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:07.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:07.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:07.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:07.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:07.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_36 2026-02-20T21:35:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:07.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:07.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:07.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:07.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:07.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:07.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:07.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:07.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_36 2026-02-20T21:35:07.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:07.967 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_36 2026-02-20T21:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '' 2026-02-20T21:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:07.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:07.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:07.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:07.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:07.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:07.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:07.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:07.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:07.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=29962], [object_number=29, tag_tid=1, entry_tid=29961], [object_number=28, tag_tid=1, entry_tid=29960], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:07.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:07.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:35:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_36 2026-02-20T21:35:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '..*' 2026-02-20T21:35:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=30, tag_tid=1, entry_tid=29962], [object_number=29, tag_tid=1, entry_tid=29961], [object_number=28, tag_tid=1, entry_tid=29960], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:08.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:08.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:08.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=30, tag_tid=1, entry_tid=29962' '!=' 'object_number=30, tag_tid=1, entry_tid=29962' 2026-02-20T21:35:08.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:08.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_36 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_36 ']' 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_36' 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_36 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_36 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:10.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_36 2026-02-20T21:35:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_36 2026-02-20T21:35:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:10.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '' 2026-02-20T21:35:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:10.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:10.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:10.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:10.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:10.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:10.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:10.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:10.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:10.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:10.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:10.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:35:10.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_36 2026-02-20T21:35:10.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:10.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:10.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:10.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_36 '..*' 2026-02-20T21:35:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:35:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_36.status 2026-02-20T21:35:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_36.status 2026-02-20T21:35:10.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:10.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:10.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_36 2026-02-20T21:35:10.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:35:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:35:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:10.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=30, tag_tid=1, entry_tid=29962], [object_number=31, tag_tid=1, entry_tid=29959]]], state=connected] 2026-02-20T21:35:10.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.status 2026-02-20T21:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_36 snap 2026-02-20T21:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:35:10.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:35:11.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_36@snap 2026-02-20T21:35:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_36': 2026-02-20T21:35:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:35:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:35:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:35:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 122213f2ae65 2026-02-20T21:35:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.122213f2ae65 2026-02-20T21:35:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:35:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:35:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:35:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:35:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:24:35 2026 2026-02-20T21:35:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:24:35 2026 2026-02-20T21:35:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:26:36 2026 2026-02-20T21:35:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:35:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 122213f2ae65 2026-02-20T21:35:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:35:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:35:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 201c4391-b079-4cdc-a105-9b7501eb934c 2026-02-20T21:35:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:35:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_36 snap 2026-02-20T21:35:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:35:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:35:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.export 2026-02-20T21:35:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_36.export 2026-02-20T21:35:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_36.export 2026-02-20T21:35:11.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_36@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.export 2026-02-20T21:35:11.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:35:11.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_36@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_36.export 2026-02-20T21:35:11.400 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:35:11.397+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:35:11.401 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:35:11.397+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:35:11.401 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:35:11.397+0000 7fd668b01640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:35:11.401 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:35:11.398+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:35:11.401 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:35:11.398+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:35:11.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:35:11.481 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_36.export 2026-02-20T21:35:11.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_36.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_36.export 2026-02-20T21:35:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:35:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_37 2026-02-20T21:35:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_37 snap 2026-02-20T21:35:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:11.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:11.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:11.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:11.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_37@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:35:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_37 2026-02-20T21:35:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_37 started 2026-02-20T21:35:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:35:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:35:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:12.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:35:13.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_37 started 2026-02-20T21:35:13.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:13.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:13.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:35:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:35:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:35:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_37 2026-02-20T21:35:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:35:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:35:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:35:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:35:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:35:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:35:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_37 2026-02-20T21:35:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:35:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:13.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:35:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:35:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:35:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:35:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:35:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:35:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:35:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:35:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:13.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:13.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:13.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:13.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_37 2026-02-20T21:35:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-20T21:35:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:35:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_37 2026-02-20T21:35:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:35:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:35:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:35:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:14.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:35:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:35:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:35:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:35:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:35:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:35:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:14.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:35:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:35:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:35:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:35:14.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:14.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_37 2026-02-20T21:35:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:35:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:35:14.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:35:14.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:35:14.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:35:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:35:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:35:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_37 2026-02-20T21:35:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:35:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:35:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:35:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:35:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_37 2026-02-20T21:35:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:35:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:35:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:35:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:35:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:35:14.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:35:14.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:14.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:35:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_37 2026-02-20T21:35:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_37 ']' 2026-02-20T21:35:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-20T21:35:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-20T21:35:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-20T21:35:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:14.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:14.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:14.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:14.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:14.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:14.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:14.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:14.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-20T21:35:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:14.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:14.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:14.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-20T21:35:14.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:14.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:14.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:14.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-20T21:35:14.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:14.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:14.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:14.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:14.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:14.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:14.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:14.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:14.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:14.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=25592], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590], [object_number=25, tag_tid=1, entry_tid=25589]]], state=connected] 2026-02-20T21:35:14.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:14.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:35:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-20T21:35:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-20T21:35:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:14.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:14.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:14.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:14.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:14.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:14.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:14.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:14.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:14.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:14.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:14.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=25592], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590], [object_number=25, tag_tid=1, entry_tid=25589]]], state=connected] 2026-02-20T21:35:14.480 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=25592' '!=' '' 2026-02-20T21:35:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:35:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=25592' = '' 2026-02-20T21:35:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:35:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:35:14.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:14.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:35:14.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:14.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:35:14.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:14.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:35:14.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:35:14.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:35:14.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:14.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:35:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:35:14.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=25592 2026-02-20T21:35:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 25592 2026-02-20T21:35:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:35:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_37 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_37 ']' 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-20T21:35:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:14.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:14.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:14.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-20T21:35:14.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:14.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:14.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-20T21:35:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-20T21:35:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:14.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:14.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:14.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=25592], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590], [object_number=25, tag_tid=1, entry_tid=25589]]], state=connected] 2026-02-20T21:35:14.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:14.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:35:14.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-20T21:35:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-20T21:35:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:14.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:14.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:14.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:14.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:14.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:14.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:14.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=25592], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590], [object_number=25, tag_tid=1, entry_tid=25589]]], state=connected] 2026-02-20T21:35:14.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=25592' '!=' 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:14.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:14.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:35:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_37 2026-02-20T21:35:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_37 ']' 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:15.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:15.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-20T21:35:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:15.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:15.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:15.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-20T21:35:15.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:15.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:15.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:15.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-20T21:35:15.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:15.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:15.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:15.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:15.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:15.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:15.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:15.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:15.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:15.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-20T21:35:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-20T21:35:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:15.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:15.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:15.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:15.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=25592], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590], [object_number=25, tag_tid=1, entry_tid=25589]]], state=connected] 2026-02-20T21:35:15.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:35:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-20T21:35:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-20T21:35:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:15.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:15.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:15.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:15.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=25592], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590], [object_number=25, tag_tid=1, entry_tid=25589]]], state=connected] 2026-02-20T21:35:15.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=25592' '!=' 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:15.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:35:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_37 2026-02-20T21:35:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:16.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_37 ']' 2026-02-20T21:35:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-20T21:35:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-20T21:35:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:16.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:16.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:16.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-20T21:35:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:16.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:16.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:16.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-20T21:35:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:16.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-20T21:35:16.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:16.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:16.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:16.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:16.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:16.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:16.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:16.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:16.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-20T21:35:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-20T21:35:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:16.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:16.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:16.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:16.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:16.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:16.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:16.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:16.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:16.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=25592], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590], [object_number=25, tag_tid=1, entry_tid=25589]]], state=connected] 2026-02-20T21:35:16.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:35:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-20T21:35:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-20T21:35:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:16.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:16.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:16.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:16.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:16.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=25592], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590], [object_number=25, tag_tid=1, entry_tid=25589]]], state=connected] 2026-02-20T21:35:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=25592' '!=' 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:16.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:35:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_37 2026-02-20T21:35:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:18.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_37 ']' 2026-02-20T21:35:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-20T21:35:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-20T21:35:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:18.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:18.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-20T21:35:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:18.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:18.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:18.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:18.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:18.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-20T21:35:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:18.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-20T21:35:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:18.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:18.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:18.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:18.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:18.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:18.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:18.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:18.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:18.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:18.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:18.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:18.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-20T21:35:18.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:18.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:18.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:18.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-20T21:35:18.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:18.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:18.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:18.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:18.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:18.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:18.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:18.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=25592], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590], [object_number=25, tag_tid=1, entry_tid=25589]]], state=connected] 2026-02-20T21:35:18.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:18.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:35:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-20T21:35:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-20T21:35:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:18.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:18.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:18.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:18.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:18.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:18.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=25592], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590], [object_number=25, tag_tid=1, entry_tid=25589]]], state=connected] 2026-02-20T21:35:18.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:18.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=25592' '!=' 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:18.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_37 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_37 ']' 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:20.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-20T21:35:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:20.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:20.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:20.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:20.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:20.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:20.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-20T21:35:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-20T21:35:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:20.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:20.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-20T21:35:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-20T21:35:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:20.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:20.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:20.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:20.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:20.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=25592], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590], [object_number=25, tag_tid=1, entry_tid=25589]]], state=connected] 2026-02-20T21:35:20.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:20.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:35:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-20T21:35:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-20T21:35:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:20.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:20.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:20.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:20.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:20.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:20.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:20.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=25592], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590], [object_number=25, tag_tid=1, entry_tid=25589]]], state=connected] 2026-02-20T21:35:20.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=25592' '!=' 'object_number=24, tag_tid=1, entry_tid=25592' 2026-02-20T21:35:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:35:22.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_37 2026-02-20T21:35:22.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:22.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_37 ']' 2026-02-20T21:35:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_37' 2026-02-20T21:35:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_37 2026-02-20T21:35:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:22.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:22.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_37 2026-02-20T21:35:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:22.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:22.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:22.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:22.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:22.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_37'] 2026-02-20T21:35:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:22.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_37 2026-02-20T21:35:22.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:22.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:22.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:22.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:22.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:22.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_37 2026-02-20T21:35:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:22.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_37 2026-02-20T21:35:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '' 2026-02-20T21:35:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:22.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:22.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:22.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:22.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:22.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:35:22.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_37 2026-02-20T21:35:22.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_37 '..*' 2026-02-20T21:35:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:35:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_37.status 2026-02-20T21:35:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_37.status 2026-02-20T21:35:22.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:22.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:22.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_37 2026-02-20T21:35:22.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:35:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:35:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=25591], [object_number=26, tag_tid=1, entry_tid=25590]]], state=connected] 2026-02-20T21:35:22.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.status 2026-02-20T21:35:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:35:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:35:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_37 snap 2026-02-20T21:35:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:22.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:35:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:35:22.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:35:23.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_37@snap 2026-02-20T21:35:23.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_37': 2026-02-20T21:35:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:35:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:35:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:35:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ff66dd085f 2026-02-20T21:35:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ff66dd085f 2026-02-20T21:35:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:35:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:35:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:35:23.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:35:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:24:45 2026 2026-02-20T21:35:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:24:45 2026 2026-02-20T21:35:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:26:47 2026 2026-02-20T21:35:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:35:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ff66dd085f 2026-02-20T21:35:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:35:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:35:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: fc6ccb28-b5bd-496f-aefe-fedb244e3e31 2026-02-20T21:35:23.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:35:23.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:23.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_37 snap 2026-02-20T21:35:23.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:23.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:35:23.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:23.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:35:23.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.export 2026-02-20T21:35:23.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_37.export 2026-02-20T21:35:23.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_37.export 2026-02-20T21:35:23.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_37@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.export 2026-02-20T21:35:23.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:35:23.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_37@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_37.export 2026-02-20T21:35:23.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:35:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_37.export 2026-02-20T21:35:23.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_37.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_37.export 2026-02-20T21:35:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:35:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_38 2026-02-20T21:35:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_38 snap 2026-02-20T21:35:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:24.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_38@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:35:25.844 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.3418502655986251) > weight (0.25) 2026-02-20T21:35:25.845 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.7090403127139759) > weight (0.25) 2026-02-20T21:35:25.845 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.5824516651338264) > weight (0.25) 2026-02-20T21:35:25.845 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.853379394918885) > weight (0.25) 2026-02-20T21:35:25.845 INFO:tasks.rbd_mirror_thrash:waiting for 110.64700485484306 secs before thrashing 2026-02-20T21:35:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_38 2026-02-20T21:35:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_38 started 2026-02-20T21:35:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:26.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:35:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:35:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:26.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:35:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_38 started 2026-02-20T21:35:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:35:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:35:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:35:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_38 2026-02-20T21:35:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:35:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:35:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:35:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:35:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:35:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:35:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:27.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_38 2026-02-20T21:35:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:35:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:35:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:27.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:35:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:35:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:35:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:35:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:35:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:35:27.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:35:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:35:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:35:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:35:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:35:27.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_38 2026-02-20T21:35:28.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:35:28.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:35:28.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:35:28.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:35:28.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:35:28.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:35:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:35:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_38 2026-02-20T21:35:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:35:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:35:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:35:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:35:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_38 2026-02-20T21:35:28.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:35:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:35:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:35:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:35:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:35:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:35:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:28.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:35:28.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-20T21:35:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-20T21:35:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-20T21:35:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-20T21:35:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:28.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:28.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:28.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:28.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-20T21:35:28.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:28.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:28.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:28.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:28.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:28.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:28.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:28.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:28.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:28.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:28.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:28.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-20T21:35:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-20T21:35:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:28.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-20T21:35:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:28.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:28.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:28.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:28.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:28.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24325], [object_number=20, tag_tid=1, entry_tid=24324], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:35:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-20T21:35:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-20T21:35:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:28.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:28.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:28.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:28.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:28.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:28.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24325], [object_number=20, tag_tid=1, entry_tid=24324], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:28.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:28.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24325' '!=' '' 2026-02-20T21:35:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:35:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24325' = '' 2026-02-20T21:35:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:28.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:35:28.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:35:28.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:28.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:35:28.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:28.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:35:28.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:28.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:35:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:35:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:35:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:35:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:35:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:28.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=24325 2026-02-20T21:35:28.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 24325 2026-02-20T21:35:28.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:35:28.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:28.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:35:28.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-20T21:35:28.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:28.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-20T21:35:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-20T21:35:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-20T21:35:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:28.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:28.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:28.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-20T21:35:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:28.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:28.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:28.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:28.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:28.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:28.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:28.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-20T21:35:28.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:28.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-20T21:35:28.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:28.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:28.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:28.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-20T21:35:28.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:28.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:28.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:28.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:28.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:28.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:28.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:28.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:28.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:28.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:28.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:28.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:28.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:28.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:28.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24325], [object_number=20, tag_tid=1, entry_tid=24324], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:28.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:28.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:35:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-20T21:35:28.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-20T21:35:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:28.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:28.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:28.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:28.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:28.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:28.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24325], [object_number=20, tag_tid=1, entry_tid=24324], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:28.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:28.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:28.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:28.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24325' '!=' 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:28.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:28.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:29.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-20T21:35:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:29.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:29.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:29.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:29.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:29.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-20T21:35:29.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-20T21:35:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-20T21:35:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:29.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:29.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:29.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:29.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:29.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24325], [object_number=20, tag_tid=1, entry_tid=24324], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:29.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:35:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-20T21:35:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-20T21:35:29.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:29.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:29.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:29.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24325], [object_number=20, tag_tid=1, entry_tid=24324], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:29.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24325' '!=' 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:29.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:30.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:30.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:30.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:30.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:30.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-20T21:35:30.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:30.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-20T21:35:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-20T21:35:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:30.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:30.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:30.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:30.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:30.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:30.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:30.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24325], [object_number=20, tag_tid=1, entry_tid=24324], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:30.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:30.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:35:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-20T21:35:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-20T21:35:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:30.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:30.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:30.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:30.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24325], [object_number=20, tag_tid=1, entry_tid=24324], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:30.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24325' '!=' 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:30.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:35:31.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-20T21:35:31.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:31.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-20T21:35:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-20T21:35:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-20T21:35:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:31.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:31.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-20T21:35:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:31.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:31.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:31.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:31.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-20T21:35:31.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:31.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-20T21:35:31.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:31.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:31.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:31.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-20T21:35:31.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:31.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:31.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:31.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24325], [object_number=20, tag_tid=1, entry_tid=24324], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:31.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:35:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-20T21:35:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-20T21:35:31.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:31.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:31.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:31.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:31.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:31.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:31.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24325], [object_number=20, tag_tid=1, entry_tid=24324], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24325' '!=' 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:35:33.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-20T21:35:33.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-20T21:35:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-20T21:35:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-20T21:35:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:33.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:33.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:33.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:33.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-20T21:35:33.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:33.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:33.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:33.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:33.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:33.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:33.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:33.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:33.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-20T21:35:33.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-20T21:35:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-20T21:35:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:33.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:33.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:33.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:33.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:33.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:34.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24325], [object_number=20, tag_tid=1, entry_tid=24324], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:34.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:35:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-20T21:35:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-20T21:35:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:34.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:34.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:34.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:34.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:34.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:34.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:34.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=24325], [object_number=20, tag_tid=1, entry_tid=24324], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:34.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=24325' '!=' 'object_number=21, tag_tid=1, entry_tid=24325' 2026-02-20T21:35:34.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:34.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:35:36.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_38 2026-02-20T21:35:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_38 ']' 2026-02-20T21:35:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_38' 2026-02-20T21:35:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_38 2026-02-20T21:35:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:36.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:36.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_38 2026-02-20T21:35:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:36.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:36.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:36.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:36.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:36.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:36.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_38 2026-02-20T21:35:36.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_38 2026-02-20T21:35:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '' 2026-02-20T21:35:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:36.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:36.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:35:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_38 2026-02-20T21:35:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_38 '..*' 2026-02-20T21:35:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:35:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_38.status 2026-02-20T21:35:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_38.status 2026-02-20T21:35:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:36.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_38 2026-02-20T21:35:36.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:35:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:35:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=24323], [object_number=22, tag_tid=1, entry_tid=24322]]], state=connected] 2026-02-20T21:35:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.status 2026-02-20T21:35:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:35:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:35:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_38 snap 2026-02-20T21:35:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:35:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:35:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:35:37.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_38@snap 2026-02-20T21:35:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_38': 2026-02-20T21:35:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:35:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:35:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:35:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11b3d633ac74 2026-02-20T21:35:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11b3d633ac74 2026-02-20T21:35:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:35:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:35:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:35:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:35:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:24:52 2026 2026-02-20T21:35:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:24:52 2026 2026-02-20T21:35:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:25:53 2026 2026-02-20T21:35:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:35:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11b3d633ac74 2026-02-20T21:35:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:35:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:35:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 68c2c064-d878-40de-847e-ee6460a0afd6 2026-02-20T21:35:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:35:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_38 snap 2026-02-20T21:35:37.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:35:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:35:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.export 2026-02-20T21:35:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_38.export 2026-02-20T21:35:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_38.export 2026-02-20T21:35:37.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_38@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.export 2026-02-20T21:35:37.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:35:37.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_38@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_38.export 2026-02-20T21:35:37.404 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:35:37.400+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:35:37.404 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:35:37.401+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:35:37.405 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:35:37.401+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:35:37.406 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:35:37.402+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:35:37.406 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:35:37.403+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:35:37.451 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:35:37.448+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:35:37.452 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:35:37.448+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:35:37.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:35:37.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_38.export 2026-02-20T21:35:37.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_38.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_38.export 2026-02-20T21:35:37.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:35:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_39 2026-02-20T21:35:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_39 snap 2026-02-20T21:35:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:37.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_39@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:35:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_39 2026-02-20T21:35:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_39 started 2026-02-20T21:35:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:35:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:35:37.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:37.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:35:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_39 started 2026-02-20T21:35:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:35:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:35:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:35:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_39 2026-02-20T21:35:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:35:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:35:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:35:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:35:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:35:38.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:35:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_39 2026-02-20T21:35:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:35:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:35:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:38.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:35:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:35:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:35:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:35:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:35:38.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:35:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:35:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:35:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:35:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:35:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:35:38.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_39 2026-02-20T21:35:39.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-20T21:35:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:35:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_39 2026-02-20T21:35:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:35:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:39.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:35:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:35:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:35:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:35:39.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:35:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:35:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:35:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:35:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:35:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:35:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:35:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:35:39.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:39.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:39.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_39 2026-02-20T21:35:39.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:35:39.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:35:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:35:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:35:39.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:35:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:35:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:35:39.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_39 2026-02-20T21:35:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:35:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:35:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:35:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:35:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_39 2026-02-20T21:35:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:35:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:39.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:35:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:35:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:35:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:35:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:35:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:39.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:35:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-20T21:35:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-20T21:35:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-20T21:35:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-20T21:35:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:39.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:39.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:39.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:39.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:39.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:39.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:39.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-20T21:35:39.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:39.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:39.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:39.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:39.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:39.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:39.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:39.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:39.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:39.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:39.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:39.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:39.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:39.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:39.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:39.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:39.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:39.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:39.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:39.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-20T21:35:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-20T21:35:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-20T21:35:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:39.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:39.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:39.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:39.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:39.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:39.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:39.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:39.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:39.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:39.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:39.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:39.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:39.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-20T21:35:39.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:39.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-20T21:35:39.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:39.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:39.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:39.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-20T21:35:39.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:39.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:39.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:39.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:39.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-20T21:35:39.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-20T21:35:39.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-20T21:35:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:39.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 3 2026-02-20T21:35:39.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 3 2026-02-20T21:35:39.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:39.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:39.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:39.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:39.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-20T21:35:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-20T21:35:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-20T21:35:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:39.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:39.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:39.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-20T21:35:39.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-20T21:35:39.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:39.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:39.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-20T21:35:39.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:39.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 3 2026-02-20T21:35:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 3 2026-02-20T21:35:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:39.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:39.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:39.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:39.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:39.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=1' '!=' '' 2026-02-20T21:35:39.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:35:39.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=1' = '' 2026-02-20T21:35:39.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:39.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=13, tag_tid=3, entry_tid=1' 2026-02-20T21:35:39.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:35:39.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=3 2026-02-20T21:35:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:35:39.706 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:39.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=2 2026-02-20T21:35:39.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=13, tag_tid=3, entry_tid=1' 2026-02-20T21:35:39.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:35:39.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:39.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:35:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:35:39.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=1 2026-02-20T21:35:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 3 = 2 -a 1 -le 1 2026-02-20T21:35:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:35:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:39.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:35:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-20T21:35:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:39.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-20T21:35:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-20T21:35:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-20T21:35:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:39.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:39.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:39.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-20T21:35:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:39.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:39.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:39.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:39.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-20T21:35:39.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-20T21:35:39.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:39.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-20T21:35:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:39.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:39.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:39.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:39.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:39.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-20T21:35:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:40.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-20T21:35:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-20T21:35:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-20T21:35:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-20T21:35:40.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-20T21:35:40.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 3 2026-02-20T21:35:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 3 2026-02-20T21:35:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:40.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:40.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:40.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-20T21:35:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-20T21:35:40.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-20T21:35:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-20T21:35:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-20T21:35:40.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-20T21:35:40.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:40.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 3 2026-02-20T21:35:40.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 3 2026-02-20T21:35:40.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:40.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:40.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:40.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=1' '!=' 'object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:40.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-20T21:35:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:40.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:40.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:40.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:40.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:40.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-20T21:35:40.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-20T21:35:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:40.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-20T21:35:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:40.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:40.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:40.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-20T21:35:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-20T21:35:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-20T21:35:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:40.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:40.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:40.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-20T21:35:40.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-20T21:35:40.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:40.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:40.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-20T21:35:40.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 3 2026-02-20T21:35:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 3 2026-02-20T21:35:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:40.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:40.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:40.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-20T21:35:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-20T21:35:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-20T21:35:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-20T21:35:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-20T21:35:40.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:40.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:40.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-20T21:35:40.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:40.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 3 2026-02-20T21:35:40.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 3 2026-02-20T21:35:40.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:40.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:40.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:40.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:40.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:40.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:40.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=1' '!=' 'object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:40.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:40.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:35:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-20T21:35:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-20T21:35:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-20T21:35:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-20T21:35:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:41.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:41.510 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-20T21:35:41.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-20T21:35:41.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:41.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:41.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:41.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:41.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:41.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:41.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:41.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:41.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:41.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:41.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:41.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:41.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:41.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-20T21:35:41.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-20T21:35:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-20T21:35:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-20T21:35:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-20T21:35:41.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:41.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:41.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-20T21:35:41.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:41.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 3 2026-02-20T21:35:41.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 3 2026-02-20T21:35:41.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:41.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:41.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:41.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-20T21:35:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-20T21:35:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-20T21:35:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:41.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:41.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:41.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-20T21:35:41.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-20T21:35:41.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:41.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:41.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-20T21:35:41.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:41.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 3 2026-02-20T21:35:41.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 3 2026-02-20T21:35:41.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:41.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:41.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:41.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:41.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:41.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=1' '!=' 'object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:41.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:35:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-20T21:35:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-20T21:35:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-20T21:35:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-20T21:35:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:43.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:43.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:43.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-20T21:35:43.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:43.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:43.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:43.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:43.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:43.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:43.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:43.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:43.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:43.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:43.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:43.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:43.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:43.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:43.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-20T21:35:43.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-20T21:35:43.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:43.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:43.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-20T21:35:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:43.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:43.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:43.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:43.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-20T21:35:43.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:43.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-20T21:35:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-20T21:35:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-20T21:35:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-20T21:35:43.406 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:43.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-20T21:35:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 3 2026-02-20T21:35:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 3 2026-02-20T21:35:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:43.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:43.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-20T21:35:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-20T21:35:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-20T21:35:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:43.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:43.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-20T21:35:43.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-20T21:35:43.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-20T21:35:43.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:43.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 3 2026-02-20T21:35:43.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 3 2026-02-20T21:35:43.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:43.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:43.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=13, tag_tid=2, entry_tid=1], [object_number=12, tag_tid=2, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:43.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=13, tag_tid=2, entry_tid=1' '!=' 'object_number=13, tag_tid=2, entry_tid=1' 2026-02-20T21:35:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:43.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_39 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_39 ']' 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_39' 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_39 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:45.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_39 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:45.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:45.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:45.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_39 2026-02-20T21:35:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_39'] 2026-02-20T21:35:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_39 2026-02-20T21:35:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:45.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:45.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:45.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_39 2026-02-20T21:35:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_39 2026-02-20T21:35:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '' 2026-02-20T21:35:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:45.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-20T21:35:45.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-20T21:35:45.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-20T21:35:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 3 2026-02-20T21:35:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 3 2026-02-20T21:35:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:45.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-20T21:35:45.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_39 2026-02-20T21:35:45.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:45.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:45.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:45.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_39 '..*' 2026-02-20T21:35:45.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:45.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:45.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:35:45.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:45.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_39.status 2026-02-20T21:35:45.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_39.status 2026-02-20T21:35:45.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:45.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:45.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_39 2026-02-20T21:35:45.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:45.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 3 2026-02-20T21:35:45.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 3 2026-02-20T21:35:45.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:45.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:45.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=13, tag_tid=3, entry_tid=1], [object_number=12, tag_tid=3, entry_tid=0], [object_number=15, tag_tid=1, entry_tid=16475], [object_number=14, tag_tid=1, entry_tid=16474]]], state=connected] 2026-02-20T21:35:45.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.status 2026-02-20T21:35:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=13, tag_tid=3, entry_tid=1' 2026-02-20T21:35:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=13, tag_tid=3, entry_tid=1' -a 'object_number=13, tag_tid=3, entry_tid=1' = 'object_number=13, tag_tid=3, entry_tid=1' 2026-02-20T21:35:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_39 snap 2026-02-20T21:35:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:35:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:35:45.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:35:46.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_39@snap 2026-02-20T21:35:46.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_39': 2026-02-20T21:35:46.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:35:46.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:35:46.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:35:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 12127ed6aaf7 2026-02-20T21:35:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.12127ed6aaf7 2026-02-20T21:35:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:35:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:35:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:35:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:35:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:24:59 2026 2026-02-20T21:35:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:24:59 2026 2026-02-20T21:35:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:27:01 2026 2026-02-20T21:35:46.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:35:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 12127ed6aaf7 2026-02-20T21:35:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:35:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:35:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: e0bd35b9-5cfb-4ecc-965c-9a8ea52b4ea0 2026-02-20T21:35:46.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:35:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:46.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_39 snap 2026-02-20T21:35:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:35:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:35:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.export 2026-02-20T21:35:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_39.export 2026-02-20T21:35:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_39.export 2026-02-20T21:35:46.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_39@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.export 2026-02-20T21:35:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:35:46.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_39@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_39.export 2026-02-20T21:35:46.845 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:35:46.842+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:35:46.845 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:35:46.842+0000 7f412fcd2640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:35:46.845 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:35:46.843+0000 7f412b4c9640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:35:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:35:46.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_39.export 2026-02-20T21:35:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_39.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_39.export 2026-02-20T21:35:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:35:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_40 2026-02-20T21:35:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_40 snap 2026-02-20T21:35:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:47.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_40@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:35:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_40 2026-02-20T21:35:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_40 started 2026-02-20T21:35:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:35:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:35:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:35:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_40 started 2026-02-20T21:35:51.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_40 2026-02-20T21:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:51.006 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:35:51.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:35:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:35:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_40 2026-02-20T21:35:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:35:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:51.011 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:35:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:35:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:35:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:35:51.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:35:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:35:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:35:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:35:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:35:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:35:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:35:51.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:35:51.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:51.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_40 2026-02-20T21:35:51.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-20T21:35:51.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:35:51.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_40 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:35:51.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:51.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:51.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:51.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:51.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_40 2026-02-20T21:35:51.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:35:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:35:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:35:51.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:35:51.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:35:51.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_40 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_40 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:51.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:35:51.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_40 2026-02-20T21:35:51.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:51.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:51.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:51.334 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_40 ']' 2026-02-20T21:35:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-20T21:35:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-20T21:35:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:51.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:51.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:51.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-20T21:35:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:51.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-20T21:35:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:51.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:51.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-02-20T21:35:51.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:51.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:51.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-20T21:35:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-20T21:35:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:51.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:51.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:51.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:51.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:51.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=46353], [object_number=44, tag_tid=1, entry_tid=46352], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:51.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:51.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:35:51.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-20T21:35:51.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:51.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:51.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-20T21:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:51.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:51.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=46353], [object_number=44, tag_tid=1, entry_tid=46352], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:51.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=46353' '!=' '' 2026-02-20T21:35:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:35:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=46353' = '' 2026-02-20T21:35:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:35:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:35:51.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:35:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:35:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:35:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:35:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:35:51.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:51.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:35:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:35:51.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:35:51.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=46353 2026-02-20T21:35:51.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 46353 2026-02-20T21:35:51.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:35:51.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:51.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:35:51.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_40 2026-02-20T21:35:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_40 ']' 2026-02-20T21:35:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-20T21:35:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-20T21:35:51.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:51.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-20T21:35:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:51.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:51.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:51.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:51.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-20T21:35:51.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:51.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:51.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-02-20T21:35:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:51.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:51.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:51.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-20T21:35:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-20T21:35:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:51.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:51.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=46353], [object_number=44, tag_tid=1, entry_tid=46352], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:51.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:35:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-20T21:35:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-20T21:35:51.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:51.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:51.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:51.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:51.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:51.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:51.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:51.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:51.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:51.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:51.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:51.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:51.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=46353], [object_number=44, tag_tid=1, entry_tid=46352], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=46353' '!=' 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:51.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:35:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_40 2026-02-20T21:35:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:52.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_40 ']' 2026-02-20T21:35:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-20T21:35:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-20T21:35:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:52.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:52.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-20T21:35:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:52.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:52.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:52.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:52.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:52.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:52.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-20T21:35:52.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:52.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-02-20T21:35:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:52.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:52.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:52.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:52.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:52.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:52.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:52.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:52.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-20T21:35:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-20T21:35:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=46353], [object_number=44, tag_tid=1, entry_tid=46352], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:52.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:35:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-20T21:35:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-20T21:35:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:52.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:52.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:52.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:52.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:52.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:52.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:52.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:52.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:52.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:52.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:52.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:52.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=46353], [object_number=44, tag_tid=1, entry_tid=46352], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:52.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:52.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=46353' '!=' 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:52.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:35:53.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_40 2026-02-20T21:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_40 ']' 2026-02-20T21:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-20T21:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-20T21:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:53.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:53.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-20T21:35:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:53.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:53.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:53.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:53.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-20T21:35:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:53.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-02-20T21:35:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:53.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:53.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:53.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:53.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:53.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:53.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-20T21:35:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-20T21:35:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:53.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:53.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:53.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=46353], [object_number=44, tag_tid=1, entry_tid=46352], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:53.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:53.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:35:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-20T21:35:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:53.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-20T21:35:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:53.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:53.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:53.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=46353], [object_number=44, tag_tid=1, entry_tid=46352], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:53.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=46353' '!=' 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:53.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:35:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_40 2026-02-20T21:35:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:55.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_40 ']' 2026-02-20T21:35:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-20T21:35:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-20T21:35:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:55.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-20T21:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:55.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:55.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:55.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:55.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:55.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:55.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-20T21:35:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:55.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-02-20T21:35:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:55.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:55.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:55.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:55.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:55.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:55.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:55.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:55.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:55.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-20T21:35:55.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:55.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:55.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:55.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-20T21:35:55.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:55.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:55.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:55.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:55.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:55.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:55.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:55.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:55.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:55.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:55.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:55.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:55.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:55.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=46353], [object_number=44, tag_tid=1, entry_tid=46352], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:55.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:35:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-20T21:35:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:55.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-20T21:35:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:55.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:55.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:55.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:55.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:55.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=46353], [object_number=44, tag_tid=1, entry_tid=46352], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:55.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:55.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=46353' '!=' 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:55.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:35:57.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_40 2026-02-20T21:35:57.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_40 ']' 2026-02-20T21:35:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-20T21:35:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-20T21:35:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:57.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:57.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:57.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:57.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:57.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-20T21:35:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:57.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:57.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:57.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:57.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:57.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:57.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:57.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:57.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-20T21:35:57.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:57.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-02-20T21:35:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:57.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:57.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:57.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:57.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:57.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:57.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:57.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:57.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:57.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:57.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:57.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:57.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:57.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-20T21:35:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-20T21:35:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:57.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:57.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:57.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:57.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=46353], [object_number=44, tag_tid=1, entry_tid=46352], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:57.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:57.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:35:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-20T21:35:57.434 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-20T21:35:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:57.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:57.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:57.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:57.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:57.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=1, entry_tid=46353], [object_number=44, tag_tid=1, entry_tid=46352], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:57.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:57.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:57.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:57.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=45, tag_tid=1, entry_tid=46353' '!=' 'object_number=45, tag_tid=1, entry_tid=46353' 2026-02-20T21:35:57.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:35:57.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:35:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_40 2026-02-20T21:35:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:35:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_40 ']' 2026-02-20T21:35:59.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_40' 2026-02-20T21:35:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_40 2026-02-20T21:35:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:35:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:35:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:35:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:35:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:35:59.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:35:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:35:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_40 2026-02-20T21:35:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:35:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:35:59.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:35:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:35:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:35:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:35:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:35:59.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:35:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:35:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:35:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:35:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:35:59.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:35:59.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:35:59.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:59.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:59.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:35:59.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:59.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_40'] 2026-02-20T21:35:59.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:35:59.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:35:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:35:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_40 2026-02-20T21:35:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:35:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:35:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:35:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:35:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:35:59.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:35:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:35:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:35:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:35:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:35:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:35:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:35:59.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:35:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:35:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:35:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:35:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:35:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:35:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:35:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:35:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:35:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:35:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:35:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:35:59.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:35:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:35:59.535 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:35:59.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:59.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:59.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:35:59.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_40 2026-02-20T21:35:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:59.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_40 2026-02-20T21:35:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '' 2026-02-20T21:35:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:35:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:59.594 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:59.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:59.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:59.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:59.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:59.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:59.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:35:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_40 2026-02-20T21:35:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_40 '..*' 2026-02-20T21:35:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:35:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:35:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:35:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:35:59.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_40.status 2026-02-20T21:35:59.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_40.status 2026-02-20T21:35:59.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:35:59.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:59.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_40 2026-02-20T21:35:59.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:59.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 11 2026-02-20T21:35:59.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 11 2026-02-20T21:35:59.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:35:59.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:59.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=45, tag_tid=2, entry_tid=1], [object_number=44, tag_tid=2, entry_tid=0], [object_number=47, tag_tid=1, entry_tid=46351], [object_number=46, tag_tid=1, entry_tid=46350]]], state=connected] 2026-02-20T21:35:59.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.status 2026-02-20T21:35:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:35:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=45, tag_tid=2, entry_tid=1' -a 'object_number=45, tag_tid=2, entry_tid=1' = 'object_number=45, tag_tid=2, entry_tid=1' 2026-02-20T21:35:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:35:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_40 snap 2026-02-20T21:35:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:35:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:35:59.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:35:59.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:36:00.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_40@snap 2026-02-20T21:36:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_40': 2026-02-20T21:36:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:36:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:36:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:36:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 122297a6a3f 2026-02-20T21:36:00.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.122297a6a3f 2026-02-20T21:36:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:36:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:36:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:36:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:36:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:25:04 2026 2026-02-20T21:36:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:25:04 2026 2026-02-20T21:36:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:28:25 2026 2026-02-20T21:36:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:36:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 122297a6a3f 2026-02-20T21:36:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:36:00.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:36:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 3ec45747-8409-4030-8a44-05569c2cec81 2026-02-20T21:36:00.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:36:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_40 snap 2026-02-20T21:36:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:36:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:36:00.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:36:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.export 2026-02-20T21:36:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_40.export 2026-02-20T21:36:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_40.export 2026-02-20T21:36:00.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_40@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.export 2026-02-20T21:36:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:36:01.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_40@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_40.export 2026-02-20T21:36:01.314 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:36:01.311+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:36:01.314 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:36:01.311+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:36:01.315 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:36:01.312+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:36:01.315 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:36:01.312+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:36:01.316 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:36:01.312+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:36:01.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:36:01.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_40.export 2026-02-20T21:36:01.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_40.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_40.export 2026-02-20T21:36:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:36:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_41 2026-02-20T21:36:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_41 snap 2026-02-20T21:36:01.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:36:01.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:01.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:01.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:36:01.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_41@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:36:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_41 2026-02-20T21:36:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_41 started 2026-02-20T21:36:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:36:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:36:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:36:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_41 started 2026-02-20T21:36:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:36:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:36:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:36:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_41 2026-02-20T21:36:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:36:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:36:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:36:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:36:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:36:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:05.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:36:05.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:05.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_41 2026-02-20T21:36:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:36:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:05.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:36:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:36:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:05.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:36:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:36:05.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:05.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:05.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:05.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:05.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_41 2026-02-20T21:36:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T21:36:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:05.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:05.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_41 2026-02-20T21:36:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:36:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:05.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:05.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:05.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:05.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:05.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:36:05.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:05.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:05.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:05.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:36:05.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:36:05.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:05.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:05.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_41 2026-02-20T21:36:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T21:36:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:05.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_41 2026-02-20T21:36:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:36:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:05.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:05.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:05.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:36:05.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:36:05.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:05.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:05.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:05.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:05.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_41 2026-02-20T21:36:05.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:36:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:36:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:36:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:36:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:36:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:36:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:36:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_41 2026-02-20T21:36:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:36:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:36:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:36:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:36:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:36:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_41 2026-02-20T21:36:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:36:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:36:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:36:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:36:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:36:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:36:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:36:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:36:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_41 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_41 ']' 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:36:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:05.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:36:05.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:05.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:36:05.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:05.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T21:36:05.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:05.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-02-20T21:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:05.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:36:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:05.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T21:36:05.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:05.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-02-20T21:36:05.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:36:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:36:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:36:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:05.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:36:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:36:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:05.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:05.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:05.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:05.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:05.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:05.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-02-20T21:36:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-02-20T21:36:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:36:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-20T21:36:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-20T21:36:05.581 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:05.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:05.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-20T21:36:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:36:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:36:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318]]], state=connected] 2026-02-20T21:36:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=42320], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318], [object_number=41, tag_tid=1, entry_tid=42317]]], state=connected] 2026-02-20T21:36:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-02-20T21:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-02-20T21:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:36:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-20T21:36:05.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-20T21:36:05.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-20T21:36:05.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:36:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318]]], state=connected] 2026-02-20T21:36:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=42320], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318], [object_number=41, tag_tid=1, entry_tid=42317]]], state=connected] 2026-02-20T21:36:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=42320' '!=' '' 2026-02-20T21:36:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:36:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=42320' = '' 2026-02-20T21:36:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:36:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:36:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:36:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:36:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:36:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:36:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:36:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:36:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:36:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:36:05.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:36:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:36:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:36:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=42320 2026-02-20T21:36:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 42320 2026-02-20T21:36:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:36:05.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:36:05.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:36:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_41 2026-02-20T21:36:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:36:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_41 ']' 2026-02-20T21:36:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-02-20T21:36:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-02-20T21:36:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:36:05.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:36:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:36:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:36:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:05.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:36:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-02-20T21:36:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:36:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:36:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:36:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:36:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:36:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:05.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:05.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:05.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:05.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T21:36:05.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:05.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-02-20T21:36:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:36:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:36:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:05.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:36:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:36:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:36:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:05.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:05.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:05.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:05.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:05.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:05.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T21:36:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:05.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:05.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:05.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-02-20T21:36:05.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:36:05.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:05.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:05.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:36:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:36:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:36:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:05.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:36:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:05.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:05.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:05.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:05.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:06.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-02-20T21:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-02-20T21:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:36:06.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-20T21:36:06.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-20T21:36:06.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:06.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:06.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:06.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-20T21:36:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:36:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:36:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318]]], state=connected] 2026-02-20T21:36:06.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=42320], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318], [object_number=41, tag_tid=1, entry_tid=42317]]], state=connected] 2026-02-20T21:36:06.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:06.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:36:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-02-20T21:36:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-02-20T21:36:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:06.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:36:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-20T21:36:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-20T21:36:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:06.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-20T21:36:06.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:36:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:36:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318]]], state=connected] 2026-02-20T21:36:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=42320], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318], [object_number=41, tag_tid=1, entry_tid=42317]]], state=connected] 2026-02-20T21:36:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=42320' '!=' 'object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:36:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_41 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_41 ']' 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:36:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:06.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:36:06.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:06.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:36:06.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:06.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-02-20T21:36:06.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:36:06.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:06.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:36:06.505 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:06.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:06.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:06.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T21:36:06.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:06.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:06.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:06.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T21:36:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:06.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:36:06.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:06.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-02-20T21:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-02-20T21:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:06.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:36:06.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-20T21:36:06.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-20T21:36:06.682 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-20T21:36:06.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:06.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:36:06.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:36:06.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:06.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318]]], state=connected] 2026-02-20T21:36:06.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=42320], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318], [object_number=41, tag_tid=1, entry_tid=42317]]], state=connected] 2026-02-20T21:36:06.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:06.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:36:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-02-20T21:36:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-02-20T21:36:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:36:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-20T21:36:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-20T21:36:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-20T21:36:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:36:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:36:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318]]], state=connected] 2026-02-20T21:36:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=42320], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318], [object_number=41, tag_tid=1, entry_tid=42317]]], state=connected] 2026-02-20T21:36:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:06.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:06.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:06.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=42320' '!=' 'object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:06.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:36:06.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:36:07.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_41 2026-02-20T21:36:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:36:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_41 ']' 2026-02-20T21:36:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-02-20T21:36:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-02-20T21:36:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:36:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:07.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:07.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:36:07.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:36:07.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:07.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:36:07.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:07.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:36:07.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:07.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-02-20T21:36:07.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:36:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:36:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:36:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:07.556 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:36:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:36:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:07.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:07.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:07.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:07.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:07.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T21:36:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:07.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:07.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:07.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-02-20T21:36:07.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:36:07.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:07.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:07.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:07.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:36:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:36:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:36:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:07.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:36:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:07.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:07.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:07.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:07.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:07.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:07.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:07.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T21:36:07.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:07.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-02-20T21:36:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:36:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:36:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:36:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:36:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:36:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:07.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:07.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:07.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-02-20T21:36:07.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:07.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:07.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:07.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-02-20T21:36:07.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:07.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:07.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:07.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:36:07.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-20T21:36:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-20T21:36:07.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-20T21:36:07.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:07.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:36:07.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:36:07.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:07.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318]]], state=connected] 2026-02-20T21:36:07.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=42320], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318], [object_number=41, tag_tid=1, entry_tid=42317]]], state=connected] 2026-02-20T21:36:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-02-20T21:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-02-20T21:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:36:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-20T21:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-20T21:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:07.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-20T21:36:07.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:36:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:36:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318]]], state=connected] 2026-02-20T21:36:07.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=40, tag_tid=1, entry_tid=42320], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318], [object_number=41, tag_tid=1, entry_tid=42317]]], state=connected] 2026-02-20T21:36:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:07.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:07.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:07.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=40, tag_tid=1, entry_tid=42320' '!=' 'object_number=40, tag_tid=1, entry_tid=42320' 2026-02-20T21:36:07.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:36:07.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:36:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_41 2026-02-20T21:36:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:09.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:36:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_41 ']' 2026-02-20T21:36:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_41' 2026-02-20T21:36:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_41 2026-02-20T21:36:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:36:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:36:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:36:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:36:09.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:09.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:36:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:09.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_41 2026-02-20T21:36:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:36:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:36:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:36:09.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:09.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:09.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:09.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:09.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T21:36:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:09.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_41 2026-02-20T21:36:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:36:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:09.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:36:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:36:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:36:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:36:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:09.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:09.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:09.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:09.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:09.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:09.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:09.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:09.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:09.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:09.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:09.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_41'] 2026-02-20T21:36:09.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:09.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_41 2026-02-20T21:36:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:36:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:09.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:36:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:36:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:36:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:36:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:09.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:09.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_41 2026-02-20T21:36:09.580 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_41 2026-02-20T21:36:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '' 2026-02-20T21:36:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:09.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:36:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-20T21:36:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-20T21:36:09.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-20T21:36:09.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:36:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:36:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318]]], state=connected] 2026-02-20T21:36:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318]]], state=connected] 2026-02-20T21:36:09.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:36:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_41 2026-02-20T21:36:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_41 '..*' 2026-02-20T21:36:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:36:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:36:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_41.status 2026-02-20T21:36:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_41.status 2026-02-20T21:36:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_41 2026-02-20T21:36:09.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 10 2026-02-20T21:36:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 10 2026-02-20T21:36:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318]]], state=connected] 2026-02-20T21:36:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=41, tag_tid=2, entry_tid=1], [object_number=40, tag_tid=2, entry_tid=0], [object_number=43, tag_tid=1, entry_tid=42319], [object_number=42, tag_tid=1, entry_tid=42318]]], state=connected] 2026-02-20T21:36:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.status 2026-02-20T21:36:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:36:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=41, tag_tid=2, entry_tid=1' -a 'object_number=41, tag_tid=2, entry_tid=1' = 'object_number=41, tag_tid=2, entry_tid=1' 2026-02-20T21:36:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_41 snap 2026-02-20T21:36:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:36:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:36:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:36:09.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:36:10.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_41@snap 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_41': 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10fffbf76267 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10fffbf76267 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:25:18 2026 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:25:18 2026 2026-02-20T21:36:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:28:21 2026 2026-02-20T21:36:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:36:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10fffbf76267 2026-02-20T21:36:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:36:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:36:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 537e716d-91d5-4e6e-b3c7-b835dbcd5a13 2026-02-20T21:36:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:36:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_41 snap 2026-02-20T21:36:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:36:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:36:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:36:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.export 2026-02-20T21:36:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_41.export 2026-02-20T21:36:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_41.export 2026-02-20T21:36:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_41@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.export 2026-02-20T21:36:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:36:10.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_41@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_41.export 2026-02-20T21:36:10.928 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:36:10.925+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:36:10.928 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:36:10.925+0000 7fd66cb09640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:36:10.928 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:36:10.925+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:36:10.929 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:36:10.926+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:36:10.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:36:10.968 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_41.export 2026-02-20T21:36:11.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_41.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_41.export 2026-02-20T21:36:11.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:36:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_42 2026-02-20T21:36:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_42 snap 2026-02-20T21:36:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:36:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:36:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:36:11.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_42@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:36:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_42 2026-02-20T21:36:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:36:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_42 started 2026-02-20T21:36:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:36:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:36:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:36:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:36:13.880 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9632074082151514) > weight (0.25) 2026-02-20T21:36:13.881 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.44210926157976516) > weight (0.25) 2026-02-20T21:36:13.881 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5397388770722058) > weight (0.25) 2026-02-20T21:36:13.881 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-20T21:36:13.881 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-20T21:36:13.881 INFO:tasks.rbd_mirror_thrash:waiting for 6.098661511750959 secs before reviving daemons 2026-02-20T21:36:13.882 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:36:13.879+0000 7feb9f379640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 116542) UID: 1000 2026-02-20T21:36:14.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_42 started 2026-02-20T21:36:14.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:14.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:14.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:36:14.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:36:14.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:36:14.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:36:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_42 2026-02-20T21:36:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:36:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:36:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:14.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:36:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:36:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:36:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:36:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_42 2026-02-20T21:36:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:36:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:14.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:14.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:36:14.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:36:14.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:14.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_42 2026-02-20T21:36:14.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:36:14.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:36:14.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:36:14.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_42 2026-02-20T21:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:36:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_42 2026-02-20T21:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:36:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:36:14.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_42 2026-02-20T21:36:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:36:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:36:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_42 ']' 2026-02-20T21:36:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-20T21:36:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-20T21:36:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:36:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:14.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:36:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:36:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:36:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:14.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:36:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-20T21:36:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:36:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:36:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:14.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:36:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:36:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:36:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:14.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:14.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:14.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-20T21:36:14.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-20T21:36:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-20T21:36:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-20T21:36:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-20T21:36:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:36:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-20T21:36:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-20T21:36:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:14.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.status 2026-02-20T21:36:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-20T21:36:14.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.status 2026-02-20T21:36:14.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:36:14.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:36:14.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:14.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38779], [object_number=38, tag_tid=1, entry_tid=38778]]], state=connected] 2026-02-20T21:36:14.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38779], [object_number=38, tag_tid=1, entry_tid=38778], [object_number=37, tag_tid=1, entry_tid=38777], [object_number=36, tag_tid=1, entry_tid=38776]]], state=connected] 2026-02-20T21:36:14.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.status 2026-02-20T21:36:14.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:36:14.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-20T21:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-20T21:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-20T21:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-20T21:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-20T21:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-20T21:36:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:14.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.status 2026-02-20T21:36:14.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-20T21:36:14.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.status 2026-02-20T21:36:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:36:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:36:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38779], [object_number=38, tag_tid=1, entry_tid=38778]]], state=connected] 2026-02-20T21:36:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=39, tag_tid=1, entry_tid=38779], [object_number=38, tag_tid=1, entry_tid=38778], [object_number=37, tag_tid=1, entry_tid=38777], [object_number=36, tag_tid=1, entry_tid=38776]]], state=connected] 2026-02-20T21:36:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.status 2026-02-20T21:36:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=39, tag_tid=1, entry_tid=38779' 2026-02-20T21:36:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=39, tag_tid=1, entry_tid=38779' 2026-02-20T21:36:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=38779' '!=' '' 2026-02-20T21:36:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:36:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=39, tag_tid=1, entry_tid=38779' = '' 2026-02-20T21:36:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=39, tag_tid=1, entry_tid=38779' 2026-02-20T21:36:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:36:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:36:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:36:14.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:36:14.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=39, tag_tid=1, entry_tid=38779' 2026-02-20T21:36:14.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:36:14.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:36:14.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:36:14.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:36:14.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:36:14.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:36:14.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:36:14.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=39, tag_tid=1, entry_tid=38779' 2026-02-20T21:36:14.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:36:14.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:36:14.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=38779 2026-02-20T21:36:14.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 38779 2026-02-20T21:36:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:36:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:36:14.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:36:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_42 2026-02-20T21:36:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:36:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:36:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_42 ']' 2026-02-20T21:36:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_42' 2026-02-20T21:36:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_42 2026-02-20T21:36:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:36:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:36:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:36:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:36:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:14.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:36:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_42 2026-02-20T21:36:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:36:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:36:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:14.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:36:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:36:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:36:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:14.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:14.640 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:14.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:14.649 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:14.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_42 2026-02-20T21:36:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:14.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_42 2026-02-20T21:36:14.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-20T21:36:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '' 2026-02-20T21:36:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-20T21:36:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:36:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-20T21:36:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-20T21:36:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.status 2026-02-20T21:36:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-20T21:36:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.status 2026-02-20T21:36:14.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:36:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:36:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38779], [object_number=38, tag_tid=1, entry_tid=38778]]], state=connected] 2026-02-20T21:36:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38779], [object_number=38, tag_tid=1, entry_tid=38778]]], state=connected] 2026-02-20T21:36:14.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.status 2026-02-20T21:36:14.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:36:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_42 2026-02-20T21:36:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-20T21:36:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_42 '..*' 2026-02-20T21:36:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-20T21:36:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:36:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_42.status 2026-02-20T21:36:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_42.status 2026-02-20T21:36:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:14.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.status 2026-02-20T21:36:14.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_42 2026-02-20T21:36:14.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.status 2026-02-20T21:36:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 9 2026-02-20T21:36:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 9 2026-02-20T21:36:14.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38779], [object_number=38, tag_tid=1, entry_tid=38778]]], state=connected] 2026-02-20T21:36:14.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=37, tag_tid=2, entry_tid=1], [object_number=36, tag_tid=2, entry_tid=0], [object_number=39, tag_tid=1, entry_tid=38779], [object_number=38, tag_tid=1, entry_tid=38778]]], state=connected] 2026-02-20T21:36:14.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.status 2026-02-20T21:36:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:36:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=37, tag_tid=2, entry_tid=1' -a 'object_number=37, tag_tid=2, entry_tid=1' = 'object_number=37, tag_tid=2, entry_tid=1' 2026-02-20T21:36:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_42 snap 2026-02-20T21:36:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:36:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:36:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:36:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:36:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:36:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_42@snap 2026-02-20T21:36:15.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_42': 2026-02-20T21:36:15.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:36:15.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:36:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:36:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11b3b9f1a91c 2026-02-20T21:36:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11b3b9f1a91c 2026-02-20T21:36:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:36:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:36:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:36:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:36:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:25:30 2026 2026-02-20T21:36:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:25:30 2026 2026-02-20T21:36:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:28:33 2026 2026-02-20T21:36:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:36:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11b3b9f1a91c 2026-02-20T21:36:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:36:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:36:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: cf755414-5772-4ccb-8e4f-4f76b327e4c6 2026-02-20T21:36:15.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:36:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_42 snap 2026-02-20T21:36:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:36:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:36:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:36:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.export 2026-02-20T21:36:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_42.export 2026-02-20T21:36:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_42.export 2026-02-20T21:36:15.801 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_42@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.export 2026-02-20T21:36:15.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:36:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_42@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_42.export 2026-02-20T21:36:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:36:16.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_42.export 2026-02-20T21:36:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_42.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_42.export 2026-02-20T21:36:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:36:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_43 2026-02-20T21:36:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_43 snap 2026-02-20T21:36:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:36:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:36:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_43@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:36:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_43 2026-02-20T21:36:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_43 started 2026-02-20T21:36:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:36:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:36:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:36:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_43 started 2026-02-20T21:36:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:36:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:36:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:36:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_43 2026-02-20T21:36:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:36:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:36:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:36:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:36:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:36:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:19.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-02-20T21:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:19.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:36:19.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:36:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:36:19.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:19.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-02-20T21:36:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2026-02-20T21:36:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:36:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:19.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:36:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:19.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:36:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:36:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:36:19.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:19.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:19.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:19.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:19.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2026-02-20T21:36:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:19.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2026-02-20T21:36:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:36:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:19.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:36:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:19.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:36:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:36:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:36:19.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:19.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:19.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2026-02-20T21:36:19.222 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:19.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_43 2026-02-20T21:36:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:36:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:36:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:36:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:36:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:36:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:36:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:36:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:19.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:36:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:36:19.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:36:19.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:19.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_43 2026-02-20T21:36:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: exception getting command descriptions: [Errno 2] No such file or directory 2026-02-20T21:36:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 1 2026-02-20T21:36:19.981 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-20T21:36:19.981 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:36:19.981 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-20T21:36:19.981 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-20T21:36:19.981 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-20T21:36:19.984 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-20T21:36:19.984 INFO:tasks.rbd_mirror_thrash:waiting for 99.50192806989969 secs before thrashing 2026-02-20T21:36:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:20.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:36:20.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:36:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:36:20.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:20.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-02-20T21:36:20.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:20.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:20.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:20.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:36:20.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:20.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2026-02-20T21:36:20.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:20.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:20.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2026-02-20T21:36:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:36:20.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:36:20.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:36:20.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2026-02-20T21:36:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_43 2026-02-20T21:36:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:36:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:36:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:36:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:36:20.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:36:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:36:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:36:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:36:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:36:20.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:36:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:20.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_43 2026-02-20T21:36:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 2 2026-02-20T21:36:22.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:22.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:36:22.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-02-20T21:36:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:36:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:22.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:22.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:36:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:36:22.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:22.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:22.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-02-20T21:36:22.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:22.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:22.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2026-02-20T21:36:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:36:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:22.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:36:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:36:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:22.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:36:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:36:22.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:22.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:22.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2026-02-20T21:36:22.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:22.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2026-02-20T21:36:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:36:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:22.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:22.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:36:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:36:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:22.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:22.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:36:22.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:36:22.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:22.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:22.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:22.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:22.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2026-02-20T21:36:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:22.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:22.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:22.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_43 2026-02-20T21:36:22.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:36:22.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:22.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:22.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:36:22.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:36:22.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:36:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:36:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:36:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:36:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:36:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:36:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:22.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:36:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:36:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:36:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:36:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:36:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:36:22.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:36:22.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:22.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_43 2026-02-20T21:36:22.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:22.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:22.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:22.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 4 2026-02-20T21:36:26.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:26.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:36:26.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:26.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-02-20T21:36:26.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:36:26.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:26.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:36:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:36:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:26.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:26.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:26.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:26.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:36:26.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:36:26.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:26.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:26.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-02-20T21:36:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:26.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:26.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2026-02-20T21:36:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:36:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:26.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:36:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:36:26.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:36:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:36:26.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:26.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2026-02-20T21:36:26.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:26.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:26.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2026-02-20T21:36:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:36:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:26.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:36:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:26.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:36:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:36:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:36:26.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:26.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2026-02-20T21:36:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_43 2026-02-20T21:36:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:36:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:36:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:36:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:36:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:36:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:36:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:36:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:36:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:36:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:36:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:36:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:36:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:36:26.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:36:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:36:26.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:36:26.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:26.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:26.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:26.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_43 2026-02-20T21:36:26.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:26.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:26.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:26.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:26.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-20T21:36:34.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:34.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:36:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-02-20T21:36:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:36:34.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:34.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:34.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:34.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:34.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:36:34.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:34.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:34.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:34.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:36:34.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:34.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:34.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:34.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:34.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:36:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:36:34.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:34.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-02-20T21:36:35.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:35.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:35.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2026-02-20T21:36:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:36:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:35.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:35.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:36:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:36:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:35.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:36:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:36:35.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:35.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2026-02-20T21:36:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:35.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:35.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2026-02-20T21:36:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:36:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:36:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:36:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:36:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:36:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:35.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:35.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2026-02-20T21:36:35.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:35.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:35.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:35.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_43 2026-02-20T21:36:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:36:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:36:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:36:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:35.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:36:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:36:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:36:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:36:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:36:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:36:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:36:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:36:35.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:36:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:36:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:36:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:36:35.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:36:35.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:35.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:35.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_43 2026-02-20T21:36:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:35.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:35.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-20T21:36:43.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:43.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:36:43.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-02-20T21:36:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:36:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:43.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:36:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:36:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:36:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:36:43.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:43.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-02-20T21:36:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:43.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2026-02-20T21:36:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:36:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:43.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:36:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:36:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:36:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:36:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:43.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:43.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:43.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2026-02-20T21:36:43.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:43.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:43.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:43.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:43.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2026-02-20T21:36:43.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:36:43.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:43.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:43.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:43.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:43.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:43.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:43.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:43.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:43.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:43.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:43.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:43.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:43.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:43.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:43.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:36:43.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:36:43.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:43.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:43.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:43.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:43.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2026-02-20T21:36:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_43 2026-02-20T21:36:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:36:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:36:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:36:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:36:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:36:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:36:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:36:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:36:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:36:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:36:43.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:36:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:43.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:43.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:36:43.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_43 2026-02-20T21:36:43.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:43.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:43.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:43.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:43.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 8 2026-02-20T21:36:51.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:36:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_43 2026-02-20T21:36:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:36:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:36:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:51.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:36:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:36:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:36:51.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:51.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_43 2026-02-20T21:36:51.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:51.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:51.513 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_43 2026-02-20T21:36:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:36:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:51.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:36:51.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:36:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:36:51.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:36:51.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:51.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:51.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_43 2026-02-20T21:36:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:51.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:36:51.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:51.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:51.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_43 2026-02-20T21:36:51.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:36:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:36:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:36:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:36:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:51.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:51.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:51.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:36:51.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:36:51.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:36:51.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:36:51.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:51.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_43 2026-02-20T21:36:51.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:36:51.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:36:51.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:36:51.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:36:51.636 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:36:51.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:36:51.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:36:51.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_43 2026-02-20T21:36:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:36:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:36:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:36:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:36:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:36:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_43 2026-02-20T21:36:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:36:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:36:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:36:51.638 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:36:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:36:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:36:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:36:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:36:51.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:36:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_43 2026-02-20T21:36:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:36:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_43 ']' 2026-02-20T21:36:51.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-20T21:36:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-20T21:36:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:36:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:36:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:36:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:36:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:36:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-20T21:36:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:36:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:36:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:36:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:36:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:36:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:51.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-02-20T21:36:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:36:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:36:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:51.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:51.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:51.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:51.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-02-20T21:36:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:36:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:36:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:36:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:51.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:36:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:36:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:51.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:51.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:51.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:51.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-20T21:36:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:52.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-20T21:36:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:36:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-20T21:36:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-20T21:36:52.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:52.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:52.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-20T21:36:52.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:36:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:36:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:52.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=23609], [object_number=20, tag_tid=1, entry_tid=23608], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:52.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:52.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:36:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-20T21:36:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:52.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-20T21:36:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:36:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-20T21:36:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-20T21:36:52.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-20T21:36:52.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:36:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:36:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=23609], [object_number=20, tag_tid=1, entry_tid=23608], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:52.282 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:52.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=23609' '!=' '' 2026-02-20T21:36:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:36:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=23609' = '' 2026-02-20T21:36:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:36:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:36:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:36:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:36:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:36:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:36:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:36:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:36:52.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:36:52.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:36:52.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:36:52.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:52.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:36:52.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:36:52.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=23609 2026-02-20T21:36:52.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 23609 2026-02-20T21:36:52.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:36:52.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:36:52.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:36:52.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_43 2026-02-20T21:36:52.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:52.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_43 ']' 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:36:52.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:52.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:36:52.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:52.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:52.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:52.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:52.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:52.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:52.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:52.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:52.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:52.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:36:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:52.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:52.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:52.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:52.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:52.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:52.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:52.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-02-20T21:36:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:36:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:36:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:52.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:52.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:52.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:52.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:52.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:52.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:52.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:52.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:52.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:52.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:52.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:52.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-20T21:36:52.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-20T21:36:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:36:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-20T21:36:52.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-20T21:36:52.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:52.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:52.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-20T21:36:52.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:36:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:36:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:52.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=23609], [object_number=20, tag_tid=1, entry_tid=23608], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:52.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:52.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:36:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-20T21:36:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-20T21:36:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:36:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-20T21:36:52.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-20T21:36:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-20T21:36:52.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:52.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:36:52.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:36:52.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:52.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:52.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=23609], [object_number=20, tag_tid=1, entry_tid=23608], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:52.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=23609' '!=' 'object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:36:52.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:36:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_43 2026-02-20T21:36:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:36:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_43 ']' 2026-02-20T21:36:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-20T21:36:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-20T21:36:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:36:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:36:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:36:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:36:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:36:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-20T21:36:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:36:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:53.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:53.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:53.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:53.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:53.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:36:53.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:53.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:53.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:53.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:53.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-20T21:36:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-20T21:36:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:36:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-20T21:36:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-20T21:36:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-20T21:36:53.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:36:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:36:53.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:53.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=23609], [object_number=20, tag_tid=1, entry_tid=23608], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:53.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:53.351 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:36:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-20T21:36:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-20T21:36:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:36:53.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-20T21:36:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-20T21:36:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:53.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-20T21:36:53.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:36:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:36:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:53.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=23609], [object_number=20, tag_tid=1, entry_tid=23608], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:53.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:53.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:53.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=23609' '!=' 'object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:36:53.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:36:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_43 2026-02-20T21:36:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:36:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_43 ']' 2026-02-20T21:36:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-20T21:36:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-20T21:36:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:36:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:36:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:36:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:36:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:36:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-20T21:36:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:36:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:36:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:36:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:36:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:54.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:36:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:54.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:54.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:54.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:54.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:54.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-02-20T21:36:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:36:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:36:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:36:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:36:54.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:36:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:54.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-02-20T21:36:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:36:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:36:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:36:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:36:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:36:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:54.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:54.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-20T21:36:54.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-20T21:36:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:36:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-20T21:36:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-20T21:36:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:54.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:54.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-20T21:36:54.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:36:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:36:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:54.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=23609], [object_number=20, tag_tid=1, entry_tid=23608], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:54.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:36:54.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-20T21:36:54.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:54.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:54.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:54.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-20T21:36:54.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:36:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-20T21:36:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-20T21:36:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-20T21:36:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:36:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:36:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:54.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=1, entry_tid=23609], [object_number=20, tag_tid=1, entry_tid=23608], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:54.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:54.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:54.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:54.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=21, tag_tid=1, entry_tid=23609' '!=' 'object_number=21, tag_tid=1, entry_tid=23609' 2026-02-20T21:36:54.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:36:54.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:36:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_43 2026-02-20T21:36:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:36:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_43 ']' 2026-02-20T21:36:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_43' 2026-02-20T21:36:56.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_43 2026-02-20T21:36:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:36:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:36:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:36:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:36:56.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:36:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:36:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_43 2026-02-20T21:36:56.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:36:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:36:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:36:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:36:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:56.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:36:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:36:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:36:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:36:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:36:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:36:56.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:36:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:36:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:36:56.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:36:56.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:56.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:56.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:36:56.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:56.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:56.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_43 2026-02-20T21:36:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:36:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:36:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:36:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:56.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:36:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:36:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:36:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:36:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:56.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:36:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:36:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:36:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:36:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:36:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:36:56.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:36:56.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:56.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:56.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:36:56.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_43'] 2026-02-20T21:36:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:36:56.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:36:56.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:36:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_43 2026-02-20T21:36:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:36:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:36:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:36:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:36:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:36:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:36:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:36:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:36:56.150 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:36:56.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:36:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:36:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:36:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:36:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:36:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:36:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:36:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:36:56.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:36:56.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:56.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:56.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:36:56.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_43 2026-02-20T21:36:56.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_43 2026-02-20T21:36:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '' 2026-02-20T21:36:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:56.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:36:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-20T21:36:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-20T21:36:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:56.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-20T21:36:56.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:36:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:36:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:56.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:56.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:36:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_43 2026-02-20T21:36:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:56.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_43 '..*' 2026-02-20T21:36:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:36:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:36:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:36:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:36:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_43.status 2026-02-20T21:36:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_43.status 2026-02-20T21:36:56.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:36:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_43 2026-02-20T21:36:56.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:56.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:36:56.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:36:56.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:36:56.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:56.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=23, tag_tid=1, entry_tid=23607], [object_number=22, tag_tid=1, entry_tid=23606]]], state=connected] 2026-02-20T21:36:56.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.status 2026-02-20T21:36:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:36:56.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:36:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_43 snap 2026-02-20T21:36:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:36:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:36:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:36:56.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:36:57.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_43@snap 2026-02-20T21:36:57.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_43': 2026-02-20T21:36:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:36:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:36:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:36:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1212a6ce8934 2026-02-20T21:36:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1212a6ce8934 2026-02-20T21:36:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:36:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:36:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:36:57.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:36:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:25:41 2026 2026-02-20T21:36:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:25:41 2026 2026-02-20T21:36:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:27:43 2026 2026-02-20T21:36:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:36:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1212a6ce8934 2026-02-20T21:36:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:36:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:36:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 3219aca3-ec83-4066-a471-3293dc89cb29 2026-02-20T21:36:57.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:36:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_43 snap 2026-02-20T21:36:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:36:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:36:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:36:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.export 2026-02-20T21:36:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_43.export 2026-02-20T21:36:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_43.export 2026-02-20T21:36:57.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_43@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.export 2026-02-20T21:36:57.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:36:57.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_43@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_43.export 2026-02-20T21:36:57.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:36:57.582 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_43.export 2026-02-20T21:36:57.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_43.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_43.export 2026-02-20T21:36:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:36:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:36:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_44 2026-02-20T21:36:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_44 snap 2026-02-20T21:36:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:36:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:36:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:36:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_44@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:36:58.182 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:36:58.179+0000 7f03c8435640 -1 reset not still connected to 0x55afceda96c0 2026-02-20T21:36:58.182 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:36:58.179+0000 7f03c8435640 -1 reset not still connected to 0x55afd799a820 2026-02-20T21:36:58.182 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:36:58.179+0000 7f03c8435640 -1 reset not still connected to 0x55afde1b5040 2026-02-20T21:36:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_44 2026-02-20T21:36:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:36:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_44 started 2026-02-20T21:36:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:36:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:36:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:36:59.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:36:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:36:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:36:59.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:37:00.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_44 started 2026-02-20T21:37:00.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:37:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:37:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:37:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:37:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_44 2026-02-20T21:37:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:37:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:37:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:37:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:37:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:37:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:37:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_44 2026-02-20T21:37:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:37:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:00.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:37:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:37:00.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:00.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:00.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:00.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:00.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_44 2026-02-20T21:37:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-20T21:37:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:00.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_44 2026-02-20T21:37:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:37:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:00.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:37:00.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:37:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:37:00.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:37:00.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:00.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_44 2026-02-20T21:37:00.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:37:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:37:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:37:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:37:00.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:37:00.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:37:00.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:37:00.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:00.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_44 2026-02-20T21:37:00.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:37:00.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:00.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:37:00.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:37:00.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:37:00.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:37:00.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_44 2026-02-20T21:37:00.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:37:00.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:00.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:37:00.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:37:00.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:37:00.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:37:00.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:37:00.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:37:00.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:00.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:37:00.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-20T21:37:00.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:00.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:00.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:37:00.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:00.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-20T21:37:00.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-20T21:37:00.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-20T21:37:00.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:00.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:00.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:00.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:00.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-20T21:37:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:00.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:00.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:00.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:00.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:00.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:00.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:00.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-20T21:37:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:00.443 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-20T21:37:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:00.446 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:00.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:00.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:00.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:00.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:00.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:00.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-20T21:37:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-20T21:37:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:00.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:00.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:00.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:00.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:00.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:00.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:00.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:00.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:00.509 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:00.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:00.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:00.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27805], [object_number=24, tag_tid=1, entry_tid=27804], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:00.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:00.539 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-20T21:37:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-20T21:37:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:00.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:00.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:00.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27805], [object_number=24, tag_tid=1, entry_tid=27804], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:00.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:00.571 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=27805' '!=' '' 2026-02-20T21:37:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:37:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=27805' = '' 2026-02-20T21:37:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:37:00.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:37:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:37:00.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:00.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:37:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:37:00.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:00.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:37:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:37:00.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=27805 2026-02-20T21:37:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 27805 2026-02-20T21:37:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:37:00.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:00.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:37:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-20T21:37:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:37:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-20T21:37:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-20T21:37:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-20T21:37:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:00.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:00.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:00.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-20T21:37:00.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:00.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:00.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:00.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:00.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:00.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:00.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:00.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:00.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:00.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:00.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:00.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:00.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:00.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:00.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:00.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:00.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:00.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:00.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:00.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:00.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:00.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:00.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-20T21:37:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:00.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-20T21:37:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-20T21:37:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-20T21:37:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:00.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:00.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:00.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:00.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:00.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:00.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:00.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27805], [object_number=24, tag_tid=1, entry_tid=27804], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:00.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:00.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-20T21:37:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-20T21:37:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:00.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:00.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:00.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:00.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:00.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:00.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:00.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27805], [object_number=24, tag_tid=1, entry_tid=27804], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:00.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=27805' '!=' 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:00.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:37:01.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-20T21:37:01.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:01.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:01.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:37:01.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:01.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-20T21:37:01.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-20T21:37:01.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-20T21:37:01.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-20T21:37:01.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:01.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:01.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:01.381 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:01.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:01.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:01.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:01.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:01.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-20T21:37:01.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:01.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:01.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:01.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-20T21:37:01.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:01.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:01.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:01.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:01.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:01.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:01.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:01.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:01.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-20T21:37:01.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:01.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:01.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:01.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-20T21:37:01.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:01.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:01.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:01.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:01.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:01.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27805], [object_number=24, tag_tid=1, entry_tid=27804], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:01.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:01.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:01.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-20T21:37:01.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:01.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:01.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:01.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-20T21:37:01.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:01.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:01.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:01.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:01.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:01.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:01.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:01.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:01.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:01.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:01.557 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:01.558 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27805], [object_number=24, tag_tid=1, entry_tid=27804], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:01.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:01.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:01.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:01.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=27805' '!=' 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:01.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:01.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:37:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-20T21:37:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:37:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-20T21:37:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-20T21:37:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-20T21:37:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-20T21:37:02.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:02.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:02.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:02.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-20T21:37:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-20T21:37:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:02.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:02.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:02.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:02.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:02.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:02.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-20T21:37:02.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:02.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-20T21:37:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:02.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:02.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:02.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:02.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:02.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27805], [object_number=24, tag_tid=1, entry_tid=27804], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:02.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:02.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-20T21:37:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-20T21:37:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:02.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:02.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:02.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:02.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:02.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:02.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:02.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:02.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:02.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27805], [object_number=24, tag_tid=1, entry_tid=27804], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:02.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:02.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:02.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=27805' '!=' 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:02.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:02.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:37:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-20T21:37:04.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:04.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:04.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:04.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:04.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:04.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:04.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:04.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:04.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-20T21:37:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:04.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:04.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-20T21:37:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:04.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:04.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:04.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:04.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:04.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-20T21:37:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-20T21:37:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:04.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:04.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:04.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27805], [object_number=24, tag_tid=1, entry_tid=27804], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:04.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:04.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-20T21:37:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-20T21:37:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:04.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:04.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:04.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:04.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27805], [object_number=24, tag_tid=1, entry_tid=27804], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:04.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=27805' '!=' 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:04.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:37:06.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-20T21:37:06.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:06.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:06.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:37:06.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:06.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-20T21:37:06.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-20T21:37:06.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-20T21:37:06.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:06.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:06.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:06.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-20T21:37:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:06.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:06.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:06.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:06.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:06.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:06.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:06.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:06.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-20T21:37:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:06.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-20T21:37:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:06.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:06.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:06.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:06.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:06.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:06.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:06.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:06.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:06.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:06.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:06.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:06.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:06.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:06.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-20T21:37:06.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:06.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:06.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:06.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-20T21:37:06.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:06.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:06.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:06.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:06.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:06.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:06.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:06.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:06.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:06.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:06.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:06.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:06.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:06.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:06.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27805], [object_number=24, tag_tid=1, entry_tid=27804], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:06.498 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:06.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-20T21:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-20T21:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:06.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:06.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=27805], [object_number=24, tag_tid=1, entry_tid=27804], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:06.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:06.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:06.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=27805' '!=' 'object_number=25, tag_tid=1, entry_tid=27805' 2026-02-20T21:37:06.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:06.533 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:37:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_44 2026-02-20T21:37:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:37:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_44 ']' 2026-02-20T21:37:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_44' 2026-02-20T21:37:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_44 2026-02-20T21:37:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:08.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_44 2026-02-20T21:37:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:08.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:08.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:08.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:08.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_44'] 2026-02-20T21:37:08.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:08.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_44 2026-02-20T21:37:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:08.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:08.598 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:08.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:08.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_44 2026-02-20T21:37:08.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:08.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_44 2026-02-20T21:37:08.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:08.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:08.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:08.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '' 2026-02-20T21:37:08.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:08.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:08.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:08.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:08.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:08.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:08.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:08.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:08.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:08.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:08.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:08.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:08.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_44 2026-02-20T21:37:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_44 '..*' 2026-02-20T21:37:08.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:37:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_44.status 2026-02-20T21:37:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_44.status 2026-02-20T21:37:08.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:08.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:08.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_44 2026-02-20T21:37:08.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:08.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:08.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=27803], [object_number=26, tag_tid=1, entry_tid=27802]]], state=connected] 2026-02-20T21:37:08.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.status 2026-02-20T21:37:08.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:08.723 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_44 snap 2026-02-20T21:37:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:37:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:37:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:37:08.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:37:09.724 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_44@snap 2026-02-20T21:37:09.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_44': 2026-02-20T21:37:09.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:37:09.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:37:09.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:37:09.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 1222f2fdd25e 2026-02-20T21:37:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.1222f2fdd25e 2026-02-20T21:37:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:37:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:37:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:37:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:37:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:25:47 2026 2026-02-20T21:37:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:25:47 2026 2026-02-20T21:37:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:27:49 2026 2026-02-20T21:37:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:37:09.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 1222f2fdd25e 2026-02-20T21:37:09.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:37:09.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:37:09.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: df41303e-35b7-46c8-9a70-c92d8f76c6f7 2026-02-20T21:37:09.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:37:09.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:09.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_44 snap 2026-02-20T21:37:09.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:09.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:37:09.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:09.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:37:09.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.export 2026-02-20T21:37:09.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_44.export 2026-02-20T21:37:09.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_44.export 2026-02-20T21:37:09.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_44@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.export 2026-02-20T21:37:09.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:37:09.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_44@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_44.export 2026-02-20T21:37:09.955 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:37:09.953+0000 7f412e4cf640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 1 2026-02-20T21:37:09.956 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:37:09.954+0000 7f412dcce640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:37:09.957 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:37:09.955+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:37:09.957 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:37:09.955+0000 7f412ecd0640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:37:09.957 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:37:09.955+0000 7f412fcd2640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 3 2026-02-20T21:37:10.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:37:10.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_44.export 2026-02-20T21:37:10.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_44.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_44.export 2026-02-20T21:37:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:37:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_45 2026-02-20T21:37:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_45 snap 2026-02-20T21:37:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:10.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_45@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:37:13.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_45 2026-02-20T21:37:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_45 started 2026-02-20T21:37:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:13.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:37:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:37:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:13.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:37:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_45 started 2026-02-20T21:37:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:37:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:37:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:37:14.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_45 2026-02-20T21:37:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:37:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:37:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:37:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:37:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:14.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:37:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:14.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:37:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_45 2026-02-20T21:37:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:37:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:14.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:37:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:37:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:14.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:37:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:37:14.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:14.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:14.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:14.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:14.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_45 2026-02-20T21:37:14.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:14.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:14.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_45 2026-02-20T21:37:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:37:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:37:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:37:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:37:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:37:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:14.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:14.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_45 2026-02-20T21:37:14.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:14.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:14.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_45 2026-02-20T21:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:14.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:37:14.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_45 2026-02-20T21:37:14.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:14.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:14.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_45 2026-02-20T21:37:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:37:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:37:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:37:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:37:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:37:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:14.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:14.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:14.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:14.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_45 2026-02-20T21:37:14.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:37:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:37:14.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:37:14.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:37:14.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:37:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:37:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:37:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_45 2026-02-20T21:37:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:37:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:14.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:37:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:37:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:37:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_45 2026-02-20T21:37:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:37:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:37:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:37:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:14.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:37:14.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:37:14.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:37:14.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:14.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:37:14.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_45 2026-02-20T21:37:14.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:14.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:14.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:14.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:14.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_45 ']' 2026-02-20T21:37:14.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-20T21:37:14.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-20T21:37:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-20T21:37:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:14.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:14.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:14.522 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:14.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:14.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:14.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:14.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:14.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:14.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:14.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:14.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:14.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2026-02-20T21:37:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:14.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:14.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:14.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:14.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:14.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:14.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:14.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:14.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:14.579 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:14.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:14.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:14.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:14.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_45 2026-02-20T21:37:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:14.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:14.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:14.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:14.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:14.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:14.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_45 2026-02-20T21:37:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:14.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:14.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:14.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:14.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:14.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:14.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:14.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:14.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-20T21:37:14.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:14.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:14.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:14.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-20T21:37:14.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:14.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:14.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:14.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:14.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-20T21:37:14.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-20T21:37:14.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-20T21:37:14.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:14.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:14.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:14.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:14.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:14.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26849], [object_number=24, tag_tid=1, entry_tid=26848], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:14.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-20T21:37:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-20T21:37:14.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-20T21:37:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-20T21:37:14.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-20T21:37:14.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:14.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26849], [object_number=24, tag_tid=1, entry_tid=26848], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:14.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=26849' '!=' '' 2026-02-20T21:37:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:37:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=26849' = '' 2026-02-20T21:37:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:37:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:37:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:37:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:37:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:37:14.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:37:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:37:14.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:14.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=26849 2026-02-20T21:37:14.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 26849 2026-02-20T21:37:14.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:37:14.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:14.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:37:15.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_45 2026-02-20T21:37:15.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:15.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:15.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:15.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_45 ']' 2026-02-20T21:37:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-20T21:37:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-20T21:37:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:15.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:15.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:15.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-20T21:37:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:15.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:15.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:15.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:15.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:15.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:15.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:15.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:15.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2026-02-20T21:37:15.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:15.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:15.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:15.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:15.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:15.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:15.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:15.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:15.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:15.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:15.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:15.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:15.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:15.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:15.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_45 2026-02-20T21:37:15.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:15.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:15.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:15.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:15.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:15.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:15.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:15.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:15.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:15.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:15.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:15.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:15.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:15.199 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_45 2026-02-20T21:37:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:15.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:15.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:15.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:15.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:15.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:15.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-20T21:37:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:15.260 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-20T21:37:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-20T21:37:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-20T21:37:15.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-20T21:37:15.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26849], [object_number=24, tag_tid=1, entry_tid=26848], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:15.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-20T21:37:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:15.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-20T21:37:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-20T21:37:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-20T21:37:15.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-20T21:37:15.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:15.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:15.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:15.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:15.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:15.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26849], [object_number=24, tag_tid=1, entry_tid=26848], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:15.323 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:15.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:15.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:15.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=26849' '!=' 'object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:15.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:15.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:37:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_45 2026-02-20T21:37:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_45 ']' 2026-02-20T21:37:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-20T21:37:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-20T21:37:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-20T21:37:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:15.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:15.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:15.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:15.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:15.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:15.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:15.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:15.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:15.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:15.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:15.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2026-02-20T21:37:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:15.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:15.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:15.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:15.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:15.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:15.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_45 2026-02-20T21:37:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:15.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:15.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:15.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:15.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:15.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:15.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_45 2026-02-20T21:37:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:15.905 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.906 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:15.907 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:15.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:15.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:15.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:15.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-20T21:37:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-20T21:37:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-20T21:37:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-20T21:37:15.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-20T21:37:15.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:15.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26849], [object_number=24, tag_tid=1, entry_tid=26848], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-20T21:37:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-20T21:37:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:16.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-20T21:37:16.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-20T21:37:16.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:16.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-20T21:37:16.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:16.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:16.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:16.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:16.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:16.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26849], [object_number=24, tag_tid=1, entry_tid=26848], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:16.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=26849' '!=' 'object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:16.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:37:16.493 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.7179469266341237) > weight (0.25) 2026-02-20T21:37:16.493 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.38774297782272626) > weight (0.25) 2026-02-20T21:37:16.493 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-20T21:37:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-20T21:37:16.494 INFO:tasks.rbd_mirror_thrash:waiting for 5.705492837024427 secs before reviving daemons 2026-02-20T21:37:16.494 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:37:16.493+0000 7fbcd310d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 92369) UID: 1000 2026-02-20T21:37:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_45 2026-02-20T21:37:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_45 ']' 2026-02-20T21:37:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-20T21:37:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-20T21:37:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:16.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:16.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:16.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-20T21:37:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:16.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:16.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:16.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:16.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:16.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:16.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:16.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:16.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:16.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:16.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:16.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2026-02-20T21:37:16.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:16.900 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:16.901 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:16.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:16.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:16.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:16.902 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:16.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:16.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_45 2026-02-20T21:37:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:16.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:16.956 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:16.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:16.965 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:16.966 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:17.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:17.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:17.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_45 2026-02-20T21:37:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:17.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:17.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:17.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:17.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:17.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:17.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-20T21:37:17.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:17.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:17.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-20T21:37:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-20T21:37:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-20T21:37:17.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:17.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:17.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-20T21:37:17.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:17.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:17.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:17.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26849], [object_number=24, tag_tid=1, entry_tid=26848], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:17.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-20T21:37:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:17.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-20T21:37:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-20T21:37:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-20T21:37:17.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-20T21:37:17.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:17.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26849], [object_number=24, tag_tid=1, entry_tid=26848], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:17.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=26849' '!=' 'object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:17.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:37:18.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_45 2026-02-20T21:37:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_45 ']' 2026-02-20T21:37:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-20T21:37:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-20T21:37:18.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:18.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:18.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:18.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-20T21:37:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:18.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:18.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:18.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:18.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:18.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:18.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:18.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2026-02-20T21:37:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:18.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:18.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:18.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:18.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:18.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:18.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_45 2026-02-20T21:37:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:18.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:18.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:18.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:18.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:18.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:18.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_45 2026-02-20T21:37:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:18.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:18.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:18.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:18.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:18.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:18.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:18.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:18.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-20T21:37:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-20T21:37:18.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-20T21:37:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-20T21:37:18.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-20T21:37:18.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:19.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:19.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26849], [object_number=24, tag_tid=1, entry_tid=26848], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:19.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:19.018 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-20T21:37:19.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-20T21:37:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-20T21:37:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-20T21:37:19.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-20T21:37:19.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:19.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:19.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:19.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:19.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:19.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=1, entry_tid=26849], [object_number=24, tag_tid=1, entry_tid=26848], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:19.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:19.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:19.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:19.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=25, tag_tid=1, entry_tid=26849' '!=' 'object_number=25, tag_tid=1, entry_tid=26849' 2026-02-20T21:37:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:19.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:37:21.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_45 2026-02-20T21:37:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_45 ']' 2026-02-20T21:37:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_45' 2026-02-20T21:37:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_45 2026-02-20T21:37:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:21.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:21.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_45 2026-02-20T21:37:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:21.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:21.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:21.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:21.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:21.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:21.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:21.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_45 2026-02-20T21:37:21.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:21.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:21.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:21.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:21.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:21.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:21.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:21.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:21.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:21.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:21.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:21.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:21.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:21.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:21.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_45 2026-02-20T21:37:21.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:21.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:21.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:21.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:21.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:21.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:21.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:21.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:21.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:21.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:21.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:21.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:21.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:21.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:21.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:21.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:21.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_45'] 2026-02-20T21:37:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:21.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_45 2026-02-20T21:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:21.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:21.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:21.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:21.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:21.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_45 2026-02-20T21:37:21.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_45 2026-02-20T21:37:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:21.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '' 2026-02-20T21:37:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-20T21:37:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-20T21:37:21.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-20T21:37:21.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:21.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:21.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:21.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:21.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:21.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:21.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:21.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_45 2026-02-20T21:37:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_45 '..*' 2026-02-20T21:37:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:21.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:37:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_45.status 2026-02-20T21:37:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_45.status 2026-02-20T21:37:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:21.329 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_45 2026-02-20T21:37:21.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:21.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:21.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:21.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:21.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:21.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=26847], [object_number=26, tag_tid=1, entry_tid=26846]]], state=connected] 2026-02-20T21:37:21.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.status 2026-02-20T21:37:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_45 snap 2026-02-20T21:37:21.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:37:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:37:21.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:37:22.200 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-20T21:37:22.200 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:37:22.200 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-20T21:37:22.201 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-20T21:37:22.201 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-20T21:37:22.203 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-20T21:37:22.203 INFO:tasks.rbd_mirror_thrash:waiting for 94.89457830336872 secs before thrashing 2026-02-20T21:37:22.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_45@snap 2026-02-20T21:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_45': 2026-02-20T21:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ff7d7cf18 2026-02-20T21:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ff7d7cf18 2026-02-20T21:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:37:22.391 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:37:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:37:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:25:54 2026 2026-02-20T21:37:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:25:54 2026 2026-02-20T21:37:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:27:56 2026 2026-02-20T21:37:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:37:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ff7d7cf18 2026-02-20T21:37:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:37:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:37:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 1aafaa0a-3f4a-4f2c-8179-9948c9b3d1be 2026-02-20T21:37:22.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:37:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_45 snap 2026-02-20T21:37:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:37:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:37:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.export 2026-02-20T21:37:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_45.export 2026-02-20T21:37:22.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_45.export 2026-02-20T21:37:22.395 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_45@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.export 2026-02-20T21:37:22.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:37:22.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_45@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_45.export 2026-02-20T21:37:22.578 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:22.576+0000 7fd66db0b640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:37:22.578 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:22.576+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:37:22.578 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:22.576+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:37:22.578 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:22.576+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:37:22.578 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:22.576+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 4 2026-02-20T21:37:22.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:37:22.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_45.export 2026-02-20T21:37:22.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_45.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_45.export 2026-02-20T21:37:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:37:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_46 2026-02-20T21:37:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_46 snap 2026-02-20T21:37:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_46@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:37:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_46 2026-02-20T21:37:25.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_46 started 2026-02-20T21:37:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:37:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:37:25.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:25.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:37:26.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_46 started 2026-02-20T21:37:26.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:26.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:26.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:37:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:37:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:37:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:37:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_46 2026-02-20T21:37:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:37:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:26.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:37:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:37:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:26.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:37:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:37:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_46 2026-02-20T21:37:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:37:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:26.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:37:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:26.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:37:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:37:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:37:26.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:26.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:26.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:26.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:26.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_46 2026-02-20T21:37:26.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T21:37:26.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:26.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_46 2026-02-20T21:37:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:37:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:26.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:26.144 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:37:26.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:26.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:26.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_46 2026-02-20T21:37:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T21:37:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:26.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:26.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:26.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_46 2026-02-20T21:37:26.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:37:26.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:26.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:26.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:37:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:37:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:26.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:37:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:37:26.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:26.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:26.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_46 2026-02-20T21:37:26.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:37:26.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:37:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:37:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:37:26.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:37:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:37:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:37:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:26.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_46 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_46 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:37:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:37:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:37:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:37:26.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_46 2026-02-20T21:37:26.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_46 ']' 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:26.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-20T21:37:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:26.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:26.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:26.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:26.482 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:26.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:26.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:26.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T21:37:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:26.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-20T21:37:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:26.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:26.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:26.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:26.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:26.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T21:37:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-02-20T21:37:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:26.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:26.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:26.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:26.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:26.597 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:26.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-20T21:37:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-20T21:37:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:26.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-20T21:37:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-20T21:37:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:26.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-20T21:37:26.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:26.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:26.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:26.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:26.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=29242], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:26.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=29242], [object_number=25, tag_tid=1, entry_tid=29241], [object_number=24, tag_tid=1, entry_tid=29240], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:26.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-20T21:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-20T21:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:26.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-20T21:37:26.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-20T21:37:26.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:26.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:26.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-20T21:37:26.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:26.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=29242], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:26.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=29242], [object_number=25, tag_tid=1, entry_tid=29241], [object_number=24, tag_tid=1, entry_tid=29240], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:26.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=29242' '!=' '' 2026-02-20T21:37:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:37:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=29242' = '' 2026-02-20T21:37:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:37:26.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:26.913 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:37:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:37:26.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:37:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:37:26.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:26.916 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:37:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:37:26.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29242 2026-02-20T21:37:26.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 29242 2026-02-20T21:37:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:37:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:26.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:37:27.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_46 2026-02-20T21:37:27.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:27.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:27.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:27.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:27.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_46 ']' 2026-02-20T21:37:27.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-20T21:37:27.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-20T21:37:27.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:27.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-20T21:37:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:27.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:27.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:27.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:27.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:27.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:27.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:27.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T21:37:27.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:27.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-20T21:37:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:27.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:27.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:27.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T21:37:27.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-02-20T21:37:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:27.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-20T21:37:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:27.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-20T21:37:27.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:27.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:27.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:27.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:27.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-20T21:37:27.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-20T21:37:27.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:27.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:27.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-20T21:37:27.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=29242], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=29242], [object_number=25, tag_tid=1, entry_tid=29241], [object_number=24, tag_tid=1, entry_tid=29240], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:27.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:27.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-20T21:37:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:27.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-20T21:37:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-20T21:37:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-20T21:37:27.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-20T21:37:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=29242], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=29242], [object_number=25, tag_tid=1, entry_tid=29241], [object_number=24, tag_tid=1, entry_tid=29240], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:27.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=29242' '!=' 'object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:27.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:37:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_46 2026-02-20T21:37:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_46 ']' 2026-02-20T21:37:27.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-20T21:37:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-20T21:37:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:27.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:27.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:27.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-20T21:37:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:27.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:27.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:27.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:27.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:27.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:27.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:27.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:27.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:27.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:27.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:27.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:27.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:27.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:27.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:27.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T21:37:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:27.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:27.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:27.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-20T21:37:27.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:27.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:27.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:27.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:27.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:27.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:27.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:27.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:27.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:27.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:27.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:27.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:27.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:27.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:27.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T21:37:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:27.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:27.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:27.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-02-20T21:37:27.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:27.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:27.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:27.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:27.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:27.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:27.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:27.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:27.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:27.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:27.903 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-20T21:37:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-20T21:37:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:27.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-20T21:37:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-20T21:37:27.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:27.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:27.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-20T21:37:27.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:27.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:27.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:27.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:27.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=29242], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:27.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=29242], [object_number=25, tag_tid=1, entry_tid=29241], [object_number=24, tag_tid=1, entry_tid=29240], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:27.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:27.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:27.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-20T21:37:27.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:27.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:27.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:27.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-20T21:37:27.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:27.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:27.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-20T21:37:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-20T21:37:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:27.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:27.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-20T21:37:27.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:28.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:28.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:28.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:28.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=29242], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:28.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=29242], [object_number=25, tag_tid=1, entry_tid=29241], [object_number=24, tag_tid=1, entry_tid=29240], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:28.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=29242' '!=' 'object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:28.015 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:37:28.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_46 2026-02-20T21:37:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_46 ']' 2026-02-20T21:37:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-20T21:37:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-20T21:37:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:28.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:28.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:28.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:28.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:28.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:28.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-20T21:37:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:28.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:28.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T21:37:28.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:28.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-20T21:37:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:28.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:28.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:28.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:28.883 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:28.891 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:28.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T21:37:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:28.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-02-20T21:37:28.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:28.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:28.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:28.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:29.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-20T21:37:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-20T21:37:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:29.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-20T21:37:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-20T21:37:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:29.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-20T21:37:29.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:29.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:29.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:29.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=29242], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:29.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=29242], [object_number=25, tag_tid=1, entry_tid=29241], [object_number=24, tag_tid=1, entry_tid=29240], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:29.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:29.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-20T21:37:29.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-20T21:37:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-20T21:37:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-20T21:37:29.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-20T21:37:29.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:29.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:29.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:29.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:29.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=29242], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:29.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=26, tag_tid=1, entry_tid=29242], [object_number=25, tag_tid=1, entry_tid=29241], [object_number=24, tag_tid=1, entry_tid=29240], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:29.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:29.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:29.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:29.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=26, tag_tid=1, entry_tid=29242' '!=' 'object_number=26, tag_tid=1, entry_tid=29242' 2026-02-20T21:37:29.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:29.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:37:30.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_46 2026-02-20T21:37:30.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:30.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_46 ']' 2026-02-20T21:37:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_46' 2026-02-20T21:37:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_46 2026-02-20T21:37:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:30.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:30.670 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_46 2026-02-20T21:37:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:30.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:30.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:30.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:30.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:30.685 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:30.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:30.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T21:37:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:30.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_46 2026-02-20T21:37:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:30.734 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:30.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:30.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:30.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:30.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:30.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:30.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_46'] 2026-02-20T21:37:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:30.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_46 2026-02-20T21:37:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:30.793 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:30.794 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:30.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_46 2026-02-20T21:37:30.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_46 2026-02-20T21:37:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '' 2026-02-20T21:37:30.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:30.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:30.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:30.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:30.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-20T21:37:30.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-20T21:37:30.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-20T21:37:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:30.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:30.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=29242], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=29242], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:30.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:30.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_46 2026-02-20T21:37:30.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:30.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_46 '..*' 2026-02-20T21:37:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:37:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_46.status 2026-02-20T21:37:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_46.status 2026-02-20T21:37:30.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_46 2026-02-20T21:37:30.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:30.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:30.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=29242], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:30.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=26, tag_tid=1, entry_tid=29242], [object_number=27, tag_tid=1, entry_tid=29239]]], state=connected] 2026-02-20T21:37:30.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.status 2026-02-20T21:37:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_46 snap 2026-02-20T21:37:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:37:30.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:37:30.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:37:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_46@snap 2026-02-20T21:37:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_46': 2026-02-20T21:37:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:37:31.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:37:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:37:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11b358c5eb43 2026-02-20T21:37:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11b358c5eb43 2026-02-20T21:37:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:37:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:37:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:37:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:37:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:26:01 2026 2026-02-20T21:37:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:26:01 2026 2026-02-20T21:37:31.951 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:28:03 2026 2026-02-20T21:37:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:37:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11b358c5eb43 2026-02-20T21:37:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:37:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:37:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: c55fb403-1702-496d-916d-e325bf1da1f3 2026-02-20T21:37:31.952 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:37:31.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:31.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_46 snap 2026-02-20T21:37:31.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:37:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:37:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.export 2026-02-20T21:37:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_46.export 2026-02-20T21:37:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_46.export 2026-02-20T21:37:31.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_46@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.export 2026-02-20T21:37:32.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:37:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_46@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_46.export 2026-02-20T21:37:32.110 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.108+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:37:32.110 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.108+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:37:32.110 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.108+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:37:32.111 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.108+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:37:32.111 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.109+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:32.152 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.150+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:37:32.153 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.151+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:32.153 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.151+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:32.153 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.151+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:37:32.178 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.176+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:37:32.178 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.176+0000 7fd667aff640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:37:32.178 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.176+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:37:32.178 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:32.176+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:37:32.208 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:37:32.206+0000 7f412acc8640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:37:32.208 INFO:tasks.ceph.cluster1.osd.1.trial069.stderr:2026-02-20T21:37:32.206+0000 7f412f4d1640 -1 bdev(0x55d26d395400 /var/lib/ceph/osd/cluster1-1/block) aio_submit retries 2 2026-02-20T21:37:32.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:37:32.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_46.export 2026-02-20T21:37:32.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_46.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_46.export 2026-02-20T21:37:32.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:32.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:37:32.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_47 2026-02-20T21:37:32.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_47 snap 2026-02-20T21:37:32.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:32.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:32.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:37:32.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:32.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_47@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:37:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_47 2026-02-20T21:37:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:35.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:37:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_47 started 2026-02-20T21:37:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:37:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:37:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:37:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:35.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:37:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_47 started 2026-02-20T21:37:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:37:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:37:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:37:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:37:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_47 2026-02-20T21:37:36.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:37:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:37:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:37:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:37:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:37:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:37:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_47 2026-02-20T21:37:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:37:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:37:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:37:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:37:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:36.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_47 2026-02-20T21:37:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-20T21:37:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:36.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:36.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_47 2026-02-20T21:37:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:37:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:36.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:37:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:37:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:37:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:37:36.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:36.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:36.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_47 2026-02-20T21:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-20T21:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:36.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_47 2026-02-20T21:37:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:37:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:36.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:37:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:37:36.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_47 2026-02-20T21:37:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:37:36.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:37:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:37:36.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:37:36.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:37:36.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:37:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:37:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_47 2026-02-20T21:37:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:37:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:37:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:37:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:37:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:37:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_47 2026-02-20T21:37:36.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:37:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:37:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:37:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:37:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:37:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:37:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:37:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:37:36.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_47 2026-02-20T21:37:36.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_47 ']' 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:36.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:36.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:36.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:36.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-20T21:37:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-20T21:37:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:36.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-20T21:37:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:36.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:36.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:36.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:36.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:36.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-20T21:37:36.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-20T21:37:36.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:36.618 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:36.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:36.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:36.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:36.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-20T21:37:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-20T21:37:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:36.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-20T21:37:36.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-20T21:37:36.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:36.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:36.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-20T21:37:36.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:36.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-20T21:37:36.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-20T21:37:36.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:36.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.status 2026-02-20T21:37:36.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-20T21:37:36.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.status 2026-02-20T21:37:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28051], [object_number=26, tag_tid=1, entry_tid=28050]]], state=connected] 2026-02-20T21:37:36.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=28052], [object_number=27, tag_tid=1, entry_tid=28051], [object_number=26, tag_tid=1, entry_tid=28050], [object_number=25, tag_tid=1, entry_tid=28049]]], state=connected] 2026-02-20T21:37:36.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.status 2026-02-20T21:37:36.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-20T21:37:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-20T21:37:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-20T21:37:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-20T21:37:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-20T21:37:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-20T21:37:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.status 2026-02-20T21:37:36.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-20T21:37:36.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.status 2026-02-20T21:37:36.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:36.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:36.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:36.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28051], [object_number=26, tag_tid=1, entry_tid=28050]]], state=connected] 2026-02-20T21:37:36.743 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=24, tag_tid=1, entry_tid=28052], [object_number=27, tag_tid=1, entry_tid=28051], [object_number=26, tag_tid=1, entry_tid=28050], [object_number=25, tag_tid=1, entry_tid=28049]]], state=connected] 2026-02-20T21:37:36.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.status 2026-02-20T21:37:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=24, tag_tid=1, entry_tid=28052' 2026-02-20T21:37:36.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=24, tag_tid=1, entry_tid=28052' 2026-02-20T21:37:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=28052' '!=' '' 2026-02-20T21:37:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:37:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=24, tag_tid=1, entry_tid=28052' = '' 2026-02-20T21:37:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=24, tag_tid=1, entry_tid=28052' 2026-02-20T21:37:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:37:36.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:36.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:37:36.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=24, tag_tid=1, entry_tid=28052' 2026-02-20T21:37:36.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:37:36.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:36.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:37:36.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:36.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:37:36.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:37:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=24, tag_tid=1, entry_tid=28052' 2026-02-20T21:37:36.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:37:36.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:36.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=28052 2026-02-20T21:37:36.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 28052 2026-02-20T21:37:36.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:37:36.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:36.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:37:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_47 2026-02-20T21:37:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:37:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_47 ']' 2026-02-20T21:37:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_47' 2026-02-20T21:37:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_47 2026-02-20T21:37:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:36.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:36.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_47 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:36.957 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:36.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:36.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:36.970 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_47 2026-02-20T21:37:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-20T21:37:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_47 2026-02-20T21:37:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:37.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:37.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:37.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:37.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:37.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:37.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:37.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:37.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_47 2026-02-20T21:37:37.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_47'] 2026-02-20T21:37:37.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:37.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_47 2026-02-20T21:37:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:37.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:37.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:37.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_47 2026-02-20T21:37:37.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_47 2026-02-20T21:37:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-20T21:37:37.141 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '' 2026-02-20T21:37:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-20T21:37:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-20T21:37:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-20T21:37:37.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:37.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.status 2026-02-20T21:37:37.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-20T21:37:37.143 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.status 2026-02-20T21:37:37.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:37.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:37.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:37.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28051], [object_number=26, tag_tid=1, entry_tid=28050]]], state=connected] 2026-02-20T21:37:37.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28051], [object_number=26, tag_tid=1, entry_tid=28050]]], state=connected] 2026-02-20T21:37:37.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.status 2026-02-20T21:37:37.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_47 2026-02-20T21:37:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-20T21:37:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_47 '..*' 2026-02-20T21:37:37.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:37.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:37.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-20T21:37:37.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:37.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_47.status 2026-02-20T21:37:37.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_47.status 2026-02-20T21:37:37.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:37.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.status 2026-02-20T21:37:37.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_47 2026-02-20T21:37:37.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.status 2026-02-20T21:37:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 6 2026-02-20T21:37:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 6 2026-02-20T21:37:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28051], [object_number=26, tag_tid=1, entry_tid=28050]]], state=connected] 2026-02-20T21:37:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=25, tag_tid=2, entry_tid=1], [object_number=24, tag_tid=2, entry_tid=0], [object_number=27, tag_tid=1, entry_tid=28051], [object_number=26, tag_tid=1, entry_tid=28050]]], state=connected] 2026-02-20T21:37:37.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.status 2026-02-20T21:37:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=25, tag_tid=2, entry_tid=1' -a 'object_number=25, tag_tid=2, entry_tid=1' = 'object_number=25, tag_tid=2, entry_tid=1' 2026-02-20T21:37:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_47 snap 2026-02-20T21:37:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:37.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:37.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:37:37.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:37.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:37:37.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:37:37.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:37:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_47@snap 2026-02-20T21:37:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_47': 2026-02-20T21:37:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:37:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:37:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:37:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 123722966363 2026-02-20T21:37:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.123722966363 2026-02-20T21:37:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:26:08 2026 2026-02-20T21:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:26:08 2026 2026-02-20T21:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:28:09 2026 2026-02-20T21:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 123722966363 2026-02-20T21:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:37:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:37:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: f879e2b6-6256-4bc2-b28f-1f122312bca8 2026-02-20T21:37:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:37:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_47 snap 2026-02-20T21:37:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.export 2026-02-20T21:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_47.export 2026-02-20T21:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_47.export 2026-02-20T21:37:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_47@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.export 2026-02-20T21:37:38.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:37:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_47@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_47.export 2026-02-20T21:37:38.403 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:38.401+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:37:38.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:37:38.523 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_47.export 2026-02-20T21:37:38.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_47.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_47.export 2026-02-20T21:37:38.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:38.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:37:38.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_48 2026-02-20T21:37:38.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_48 snap 2026-02-20T21:37:38.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:38.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:38.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:38.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:38.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_48@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:37:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_48 2026-02-20T21:37:41.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_48 started 2026-02-20T21:37:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:37:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:37:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:41.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:37:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_48 started 2026-02-20T21:37:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:37:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:37:42.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:37:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_48 2026-02-20T21:37:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:37:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:37:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:37:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:37:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:42.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:37:42.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:42.096 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:37:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_48 2026-02-20T21:37:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:37:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:42.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:37:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:42.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:37:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:37:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:37:42.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:42.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:42.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_48 2026-02-20T21:37:42.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:42.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:42.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_48 2026-02-20T21:37:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:37:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:42.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:42.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:37:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:37:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:37:42.158 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:42.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:42.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:42.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:42.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_48 2026-02-20T21:37:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:42.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_48 2026-02-20T21:37:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:37:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:42.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:42.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:37:42.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:37:42.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:42.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:42.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:42.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:42.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_48 2026-02-20T21:37:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:42.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_48 2026-02-20T21:37:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:37:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:42.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:37:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:37:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:42.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:37:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:37:42.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:42.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_48 2026-02-20T21:37:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:37:42.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:37:42.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:37:42.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:37:42.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:37:42.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:37:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:37:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_48 2026-02-20T21:37:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:37:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:37:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:37:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:37:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_48 2026-02-20T21:37:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:37:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:37:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:37:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:37:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:37:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:37:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:37:42.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_48 2026-02-20T21:37:42.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_48 ']' 2026-02-20T21:37:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_48' 2026-02-20T21:37:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2026-02-20T21:37:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:42.541 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:42.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2026-02-20T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:42.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:42.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:42.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:42.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:42.596 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2026-02-20T21:37:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:42.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:42.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:42.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:42.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:42.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:42.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:42.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_48 2026-02-20T21:37:42.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:42.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:42.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:42.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:42.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:42.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:42.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:42.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:42.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:42.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:42.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:42.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:42.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:42.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:42.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:42.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:42.711 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:42.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_48 2026-02-20T21:37:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:42.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:42.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:42.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:42.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:42.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_48 2026-02-20T21:37:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:42.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '' 2026-02-20T21:37:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-20T21:37:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-20T21:37:42.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:42.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:42.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-20T21:37:42.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242]]], state=connected] 2026-02-20T21:37:42.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242], [object_number=29, tag_tid=1, entry_tid=29241], [object_number=28, tag_tid=1, entry_tid=29240]]], state=connected] 2026-02-20T21:37:42.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:42.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:42.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2026-02-20T21:37:42.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:42.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2026-02-20T21:37:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-20T21:37:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-20T21:37:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:42.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:42.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-20T21:37:42.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:42.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:42.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:42.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:42.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242]]], state=connected] 2026-02-20T21:37:42.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242], [object_number=29, tag_tid=1, entry_tid=29241], [object_number=28, tag_tid=1, entry_tid=29240]]], state=connected] 2026-02-20T21:37:42.836 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:42.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:42.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:42.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=29243' '!=' '' 2026-02-20T21:37:42.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:37:42.838 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=29243' = '' 2026-02-20T21:37:42.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:42.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:42.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:37:42.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:42.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:37:42.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:37:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:42.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:37:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:37:42.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:42.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:37:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:37:42.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=29243 2026-02-20T21:37:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 29243 2026-02-20T21:37:42.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:37:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:42.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:37:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_48 2026-02-20T21:37:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_48 ']' 2026-02-20T21:37:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_48' 2026-02-20T21:37:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2026-02-20T21:37:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2026-02-20T21:37:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:43.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:43.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:43.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:43.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2026-02-20T21:37:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:43.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:43.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:43.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:43.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:43.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:43.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:43.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_48 2026-02-20T21:37:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:43.166 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:43.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:43.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:43.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_48 2026-02-20T21:37:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:43.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:43.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:43.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:43.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_48 2026-02-20T21:37:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '' 2026-02-20T21:37:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:43.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-20T21:37:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-20T21:37:43.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-20T21:37:43.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242]]], state=connected] 2026-02-20T21:37:43.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242], [object_number=29, tag_tid=1, entry_tid=29241], [object_number=28, tag_tid=1, entry_tid=29240]]], state=connected] 2026-02-20T21:37:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:43.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:43.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2026-02-20T21:37:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2026-02-20T21:37:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-20T21:37:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-20T21:37:43.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:43.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:43.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-20T21:37:43.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:43.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:43.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:43.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:43.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242]]], state=connected] 2026-02-20T21:37:43.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242], [object_number=29, tag_tid=1, entry_tid=29241], [object_number=28, tag_tid=1, entry_tid=29240]]], state=connected] 2026-02-20T21:37:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=29243' '!=' 'object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:43.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:37:43.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_48 2026-02-20T21:37:43.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_48 ']' 2026-02-20T21:37:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_48' 2026-02-20T21:37:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2026-02-20T21:37:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2026-02-20T21:37:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:43.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:43.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:43.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:43.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:43.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:43.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:43.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:43.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:43.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2026-02-20T21:37:43.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:43.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:43.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:43.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:43.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:43.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:43.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:43.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_48 2026-02-20T21:37:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:43.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:43.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:43.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:43.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:43.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_48 2026-02-20T21:37:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:43.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:43.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:43.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:43.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:43.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:43.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:43.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_48 2026-02-20T21:37:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '' 2026-02-20T21:37:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:43.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-20T21:37:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-20T21:37:43.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:43.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:43.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-20T21:37:43.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242]]], state=connected] 2026-02-20T21:37:44.019 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242], [object_number=29, tag_tid=1, entry_tid=29241], [object_number=28, tag_tid=1, entry_tid=29240]]], state=connected] 2026-02-20T21:37:44.020 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:44.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2026-02-20T21:37:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:44.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2026-02-20T21:37:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-20T21:37:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-20T21:37:44.023 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:44.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:44.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-20T21:37:44.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242]]], state=connected] 2026-02-20T21:37:44.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242], [object_number=29, tag_tid=1, entry_tid=29241], [object_number=28, tag_tid=1, entry_tid=29240]]], state=connected] 2026-02-20T21:37:44.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=29243' '!=' 'object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:37:44.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_48 2026-02-20T21:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_48 ']' 2026-02-20T21:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_48' 2026-02-20T21:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2026-02-20T21:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:44.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:44.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2026-02-20T21:37:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:44.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:44.862 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:44.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:44.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2026-02-20T21:37:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:44.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:44.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:44.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:44.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:44.928 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:44.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:44.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:44.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_48 2026-02-20T21:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:44.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:45.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:45.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:45.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_48 2026-02-20T21:37:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:45.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:45.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_48 2026-02-20T21:37:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '' 2026-02-20T21:37:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:45.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-20T21:37:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-20T21:37:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:45.092 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:45.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-20T21:37:45.093 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:45.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:45.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242]]], state=connected] 2026-02-20T21:37:45.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242], [object_number=29, tag_tid=1, entry_tid=29241], [object_number=28, tag_tid=1, entry_tid=29240]]], state=connected] 2026-02-20T21:37:45.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2026-02-20T21:37:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:45.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2026-02-20T21:37:45.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:45.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:45.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:45.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:45.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-20T21:37:45.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-20T21:37:45.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-20T21:37:45.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:45.153 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242]]], state=connected] 2026-02-20T21:37:45.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242], [object_number=29, tag_tid=1, entry_tid=29241], [object_number=28, tag_tid=1, entry_tid=29240]]], state=connected] 2026-02-20T21:37:45.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:45.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=29243' '!=' 'object_number=31, tag_tid=1, entry_tid=29243' 2026-02-20T21:37:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:45.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:37:46.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_48 2026-02-20T21:37:46.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:46.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:46.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:46.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:46.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_48 ']' 2026-02-20T21:37:46.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_48' 2026-02-20T21:37:46.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_48 2026-02-20T21:37:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:46.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_48 2026-02-20T21:37:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:46.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:46.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:46.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_48 2026-02-20T21:37:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:46.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:46.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:46.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:46.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:46.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:46.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:46.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:46.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:46.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:46.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_48 2026-02-20T21:37:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:46.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:46.878 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:46.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:46.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:46.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:46.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_48'] 2026-02-20T21:37:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:46.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:46.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_48 2026-02-20T21:37:46.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:46.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:46.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:46.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:46.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:46.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:46.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:46.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:46.937 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:46.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:46.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:46.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:46.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:46.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_48 2026-02-20T21:37:47.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_48 2026-02-20T21:37:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '' 2026-02-20T21:37:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:47.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-20T21:37:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-20T21:37:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:47.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-20T21:37:47.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242]]], state=connected] 2026-02-20T21:37:47.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242]]], state=connected] 2026-02-20T21:37:47.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:47.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_48 2026-02-20T21:37:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_48 '..*' 2026-02-20T21:37:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:37:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:47.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_48.status 2026-02-20T21:37:47.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_48.status 2026-02-20T21:37:47.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:47.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:47.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_48 2026-02-20T21:37:47.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242]]], state=connected] 2026-02-20T21:37:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=29243], [object_number=30, tag_tid=1, entry_tid=29242]]], state=connected] 2026-02-20T21:37:47.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.status 2026-02-20T21:37:47.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:47.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:47.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:47.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_48 snap 2026-02-20T21:37:47.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:47.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:47.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:47.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:37:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:37:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:37:48.273 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_48@snap 2026-02-20T21:37:48.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_48': 2026-02-20T21:37:48.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:37:48.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:37:48.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:37:48.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 123732aa8a3a 2026-02-20T21:37:48.303 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.123732aa8a3a 2026-02-20T21:37:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:37:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:37:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:37:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:37:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:26:18 2026 2026-02-20T21:37:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:26:18 2026 2026-02-20T21:37:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:28:20 2026 2026-02-20T21:37:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:37:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 123732aa8a3a 2026-02-20T21:37:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:37:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:37:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 8b83aab9-140e-43c2-86aa-451c58889869 2026-02-20T21:37:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:37:48.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:48.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_48 snap 2026-02-20T21:37:48.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:48.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:37:48.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:48.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:37:48.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.export 2026-02-20T21:37:48.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_48.export 2026-02-20T21:37:48.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_48.export 2026-02-20T21:37:48.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_48@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.export 2026-02-20T21:37:48.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:37:48.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_48@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_48.export 2026-02-20T21:37:48.464 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.462+0000 7fd66ab05640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:37:48.464 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.463+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:37:48.465 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.463+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:37:48.465 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.464+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:37:48.504 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.502+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:48.504 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.502+0000 7fd66bb07640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:48.504 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.502+0000 7fd669302640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:48.504 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.502+0000 7fd66b306640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:48.504 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.502+0000 7fd669b03640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:48.541 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.539+0000 7fd66a304640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:48.542 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.540+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:48.542 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.540+0000 7fd6672fe640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:48.542 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.540+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:48.542 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.540+0000 7fd668300640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:48.542 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:48.540+0000 7fd666afd640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 3 2026-02-20T21:37:48.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:37:48.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_48.export 2026-02-20T21:37:48.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_48.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_48.export 2026-02-20T21:37:48.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:37:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_49 2026-02-20T21:37:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_49 snap 2026-02-20T21:37:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:48.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_49@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:37:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_49 2026-02-20T21:37:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:51.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_49 started 2026-02-20T21:37:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:37:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:37:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:51.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:37:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_49 started 2026-02-20T21:37:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:37:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:37:52.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:37:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_49 2026-02-20T21:37:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:37:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:37:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:37:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:37:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:52.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:37:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:37:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_49 2026-02-20T21:37:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:37:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:37:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:52.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:37:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:37:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:37:52.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_49 2026-02-20T21:37:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_49 2026-02-20T21:37:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:37:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:37:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:37:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:37:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:37:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:52.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:52.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:52.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:52.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_49 2026-02-20T21:37:52.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:52.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:52.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/image_49 2026-02-20T21:37:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:2 2026-02-20T21:37:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:52.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:2 2026-02-20T21:37:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:52.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:2 2026-02-20T21:37:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=2 2026-02-20T21:37:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=2 2026-02-20T21:37:52.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/image_49 2026-02-20T21:37:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:37:52.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/image_49 2026-02-20T21:37:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:3 2026-02-20T21:37:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:37:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:52.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:3 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:37:52.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:3 2026-02-20T21:37:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:37:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:37:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:37:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:37:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:37:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=3 2026-02-20T21:37:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=3 2026-02-20T21:37:52.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:52.294 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:52.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:52.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:52.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/image_49 2026-02-20T21:37:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:37:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:37:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:37:52.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:37:52.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:37:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:37:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:37:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:52.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_49 2026-02-20T21:37:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:37:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:37:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:37:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:37:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_49 2026-02-20T21:37:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:37:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:37:52.347 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:37:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:37:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:37:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:37:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:52.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:37:52.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_49 2026-02-20T21:37:52.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:52.547 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:52.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:52.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:52.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_49 ']' 2026-02-20T21:37:52.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-20T21:37:52.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-20T21:37:52.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:52.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:52.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:52.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:52.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:52.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:52.548 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:52.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:52.549 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:52.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:52.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-20T21:37:52.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:52.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:52.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:52.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:52.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:52.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:52.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:52.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:52.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:52.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:52.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:52.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:52.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:52.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:52.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:52.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:52.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:52.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:52.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:52.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:52.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-20T21:37:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:52.606 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:52.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:52.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:52.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:52.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:52.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:52.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:52.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:52.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:52.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:52.607 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:52.608 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:52.609 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:52.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:52.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:52.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:52.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:52.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:52.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_49 2026-02-20T21:37:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:52.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:52.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:52.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:52.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:52.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:52.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:52.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:52.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:52.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:52.722 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:52.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:52.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_49 2026-02-20T21:37:52.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:52.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:52.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:52.725 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:52.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:52.727 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:52.728 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:52.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-20T21:37:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-20T21:37:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-20T21:37:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-20T21:37:52.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-20T21:37:52.787 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466]]], state=connected] 2026-02-20T21:37:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466], [object_number=29, tag_tid=1, entry_tid=31465], [object_number=28, tag_tid=1, entry_tid=31464]]], state=connected] 2026-02-20T21:37:52.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:52.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:52.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-20T21:37:52.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:52.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:52.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:52.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-20T21:37:52.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:52.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:52.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:52.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:52.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-20T21:37:52.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-20T21:37:52.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-20T21:37:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466]]], state=connected] 2026-02-20T21:37:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466], [object_number=29, tag_tid=1, entry_tid=31465], [object_number=28, tag_tid=1, entry_tid=31464]]], state=connected] 2026-02-20T21:37:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:52.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:52.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:52.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=31467' '!=' '' 2026-02-20T21:37:52.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:37:52.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=31467' = '' 2026-02-20T21:37:52.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:52.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:52.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:37:52.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:52.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:37:52.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:52.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:37:52.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:52.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:37:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:37:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:52.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:37:52.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:52.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:37:52.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:37:52.858 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=31467 2026-02-20T21:37:52.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 31467 2026-02-20T21:37:52.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:37:52.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:52.859 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:37:53.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_49 2026-02-20T21:37:53.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:53.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:53.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:53.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:53.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_49 ']' 2026-02-20T21:37:53.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-20T21:37:53.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-20T21:37:53.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:53.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:53.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:53.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:53.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:53.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:53.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:53.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:53.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:53.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-20T21:37:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:53.065 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:53.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:53.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:53.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:53.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:53.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:53.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-20T21:37:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:53.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:53.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:53.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:53.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:53.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:53.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_49 2026-02-20T21:37:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:53.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:53.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:53.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_49 2026-02-20T21:37:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:53.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-20T21:37:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-20T21:37:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:53.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:53.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:53.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:53.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-20T21:37:53.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-20T21:37:53.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:53.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:53.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-20T21:37:53.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:53.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:53.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:53.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:53.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466]]], state=connected] 2026-02-20T21:37:53.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466], [object_number=29, tag_tid=1, entry_tid=31465], [object_number=28, tag_tid=1, entry_tid=31464]]], state=connected] 2026-02-20T21:37:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:53.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-20T21:37:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-20T21:37:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:53.331 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:53.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-20T21:37:53.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-20T21:37:53.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:53.332 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-20T21:37:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:53.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466]]], state=connected] 2026-02-20T21:37:53.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466], [object_number=29, tag_tid=1, entry_tid=31465], [object_number=28, tag_tid=1, entry_tid=31464]]], state=connected] 2026-02-20T21:37:53.361 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:53.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:53.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:53.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=31467' '!=' 'object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:53.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:37:53.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_49 2026-02-20T21:37:53.764 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_49 ']' 2026-02-20T21:37:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-20T21:37:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-20T21:37:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:53.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:53.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-20T21:37:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:53.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:53.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:53.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:53.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:53.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:53.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:53.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:53.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:53.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-20T21:37:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:53.825 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:53.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:53.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:53.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:53.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:53.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:53.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:53.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:53.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:53.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:53.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:53.835 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:53.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:53.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_49 2026-02-20T21:37:53.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:53.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:53.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:53.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:53.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:53.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:53.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:53.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:53.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:53.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:53.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:53.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:53.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:53.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:53.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_49 2026-02-20T21:37:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:53.950 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:53.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-20T21:37:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-20T21:37:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:53.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-20T21:37:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-20T21:37:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:54.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-20T21:37:54.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:54.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:54.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:54.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:54.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466]]], state=connected] 2026-02-20T21:37:54.028 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466], [object_number=29, tag_tid=1, entry_tid=31465], [object_number=28, tag_tid=1, entry_tid=31464]]], state=connected] 2026-02-20T21:37:54.029 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:54.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-20T21:37:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-20T21:37:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:54.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-20T21:37:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-20T21:37:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:54.033 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-20T21:37:54.034 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:54.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:54.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:54.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:54.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466]]], state=connected] 2026-02-20T21:37:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466], [object_number=29, tag_tid=1, entry_tid=31465], [object_number=28, tag_tid=1, entry_tid=31464]]], state=connected] 2026-02-20T21:37:54.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:54.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:54.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:54.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=31467' '!=' 'object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:54.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:54.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:37:54.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_49 2026-02-20T21:37:54.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_49 ']' 2026-02-20T21:37:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-20T21:37:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-20T21:37:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:54.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:54.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:54.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-20T21:37:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:54.869 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:54.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:54.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:54.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:54.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:54.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:54.872 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:54.879 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:54.921 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:54.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-20T21:37:54.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:54.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:54.926 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:54.927 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:54.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:54.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:54.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:54.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:54.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:54.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:54.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:54.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:54.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_49 2026-02-20T21:37:54.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:54.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:54.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:54.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:54.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:54.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:54.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:54.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:54.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:54.982 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:54.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:54.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:54.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:54.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:54.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:54.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:54.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:54.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:54.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:54.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:54.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:54.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:54.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:54.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:54.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:54.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:54.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:54.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:54.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:54.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:54.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:54.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:54.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:54.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:55.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_49 2026-02-20T21:37:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:55.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:55.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:55.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:55.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:55.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:55.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:55.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:55.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:55.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:55.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-20T21:37:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-20T21:37:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:55.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-20T21:37:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-20T21:37:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:55.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-20T21:37:55.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466]]], state=connected] 2026-02-20T21:37:55.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466], [object_number=29, tag_tid=1, entry_tid=31465], [object_number=28, tag_tid=1, entry_tid=31464]]], state=connected] 2026-02-20T21:37:55.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:55.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-20T21:37:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-20T21:37:55.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-20T21:37:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-20T21:37:55.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-20T21:37:55.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466]]], state=connected] 2026-02-20T21:37:55.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466], [object_number=29, tag_tid=1, entry_tid=31465], [object_number=28, tag_tid=1, entry_tid=31464]]], state=connected] 2026-02-20T21:37:55.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=31, tag_tid=1, entry_tid=31467' '!=' 'object_number=31, tag_tid=1, entry_tid=31467' 2026-02-20T21:37:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:37:55.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:37:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_49 2026-02-20T21:37:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:56.767 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:37:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_49 ']' 2026-02-20T21:37:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_49' 2026-02-20T21:37:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_49 2026-02-20T21:37:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:37:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:37:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:37:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:37:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:37:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:37:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_49 2026-02-20T21:37:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:37:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:37:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:37:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:37:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:56.772 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:37:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:37:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:37:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:37:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:37:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:37:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:37:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:37:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:37:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:37:56.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:56.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:56.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:37:56.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:56.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:56.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:56.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:56.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:56.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_49 2026-02-20T21:37:56.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:37:56.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:56.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:56.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:37:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:37:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:37:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:37:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:37:56.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:37:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:37:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:37:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:37:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:37:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:56.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:37:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:37:56.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:37:56.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:37:56.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:56.884 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:2 rbd mirror flush mirror/image_49 2026-02-20T21:37:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:37:56.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-20T21:37:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:2 2026-02-20T21:37:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=2 2026-02-20T21:37:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:56.888 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=2 2026-02-20T21:37:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-20T21:37:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:2 2026-02-20T21:37:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-20T21:37:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:2 2026-02-20T21:37:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=2 2026-02-20T21:37:56.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-20T21:37:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=2 2026-02-20T21:37:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=2 2026-02-20T21:37:56.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-20T21:37:56.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:56.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:56.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-20T21:37:56.899 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:56.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_49'] 2026-02-20T21:37:56.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:37:56.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:37:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:37:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:3 rbd mirror flush mirror/image_49 2026-02-20T21:37:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:37:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:37:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:37:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-20T21:37:56.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:3 2026-02-20T21:37:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:37:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:37:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:37:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=3 2026-02-20T21:37:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:37:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=3 2026-02-20T21:37:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:56.946 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-20T21:37:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:3 2026-02-20T21:37:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:37:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:37:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-20T21:37:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:3 2026-02-20T21:37:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:37:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:37:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:37:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=3 2026-02-20T21:37:56.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-20T21:37:56.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:37:56.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:37:56.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=3 2026-02-20T21:37:56.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=3 2026-02-20T21:37:56.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-20T21:37:56.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:56.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:56.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-20T21:37:56.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush mirror/image_49 2026-02-20T21:37:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_49 2026-02-20T21:37:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:57.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '' 2026-02-20T21:37:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:37:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-20T21:37:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-20T21:37:57.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:57.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:57.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-20T21:37:57.010 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466]]], state=connected] 2026-02-20T21:37:57.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466]]], state=connected] 2026-02-20T21:37:57.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:57.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_49 2026-02-20T21:37:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_49 '..*' 2026-02-20T21:37:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:37:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:37:57.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:37:57.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:37:57.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_49.status 2026-02-20T21:37:57.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_49.status 2026-02-20T21:37:57.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:37:57.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:57.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_49 2026-02-20T21:37:57.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:57.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 7 2026-02-20T21:37:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 7 2026-02-20T21:37:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:37:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466]]], state=connected] 2026-02-20T21:37:57.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=29, tag_tid=2, entry_tid=1], [object_number=28, tag_tid=2, entry_tid=0], [object_number=31, tag_tid=1, entry_tid=31467], [object_number=30, tag_tid=1, entry_tid=31466]]], state=connected] 2026-02-20T21:37:57.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.status 2026-02-20T21:37:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=29, tag_tid=2, entry_tid=1' -a 'object_number=29, tag_tid=2, entry_tid=1' = 'object_number=29, tag_tid=2, entry_tid=1' 2026-02-20T21:37:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:57.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_49 snap 2026-02-20T21:37:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:37:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:37:57.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:37:58.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_49@snap 2026-02-20T21:37:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_49': 2026-02-20T21:37:58.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:37:58.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:37:58.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:37:58.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 10ff20e6a518 2026-02-20T21:37:58.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.10ff20e6a518 2026-02-20T21:37:58.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:37:58.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:37:58.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:37:58.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:37:58.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:26:26 2026 2026-02-20T21:37:58.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:26:26 2026 2026-02-20T21:37:58.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:28:28 2026 2026-02-20T21:37:58.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:37:58.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 10ff20e6a518 2026-02-20T21:37:58.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:37:58.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:37:58.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 1809784e-db84-4903-8eed-eecec56c5f22 2026-02-20T21:37:58.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:37:58.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_49 snap 2026-02-20T21:37:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:37:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:37:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.export 2026-02-20T21:37:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_49.export 2026-02-20T21:37:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_49.export 2026-02-20T21:37:58.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_49@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.export 2026-02-20T21:37:58.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:37:58.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_49@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_49.export 2026-02-20T21:37:58.303 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:58.301+0000 7fd6662fc640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:37:58.303 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:58.301+0000 7fd66c308640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 1 2026-02-20T21:37:58.304 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:37:58.302+0000 7fd66d30a640 -1 bdev(0x55daad491400 /var/lib/ceph/osd/cluster1-0/block) aio_submit retries 2 2026-02-20T21:37:58.371 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:37:58.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_49.export 2026-02-20T21:37:58.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_49.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_49.export 2026-02-20T21:37:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:37:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:37:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_50 2026-02-20T21:37:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_snap cluster2 mirror image_50 snap 2026-02-20T21:37:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:37:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:58.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:37:58.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:37:58.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror snap create image_50@snap --debug-rbd=20 --debug-journaler=20 2026-02-20T21:37:59.486 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.7549919628366596) > weight (0.25) 2026-02-20T21:37:59.487 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.4069390512199329) > weight (0.25) 2026-02-20T21:37:59.487 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.8123362749365045) > weight (0.25) 2026-02-20T21:37:59.487 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.5799619671156357) > weight (0.25) 2026-02-20T21:37:59.487 INFO:tasks.rbd_mirror_thrash:waiting for 95.90258486186181 secs before thrashing 2026-02-20T21:37:59.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_started cluster1 mirror image_50 2026-02-20T21:37:59.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:59.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:37:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_replay_state cluster1 mirror image_50 started 2026-02-20T21:37:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:37:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:37:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:37:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=started 2026-02-20T21:37:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:37:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:37:59.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:38:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_replay_state cluster1 mirror image_50 started 2026-02-20T21:38:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:00.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=started 2026-02-20T21:38:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local status_result 2026-02-20T21:38:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=stopped 2026-02-20T21:38:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemons cluster1 rbd mirror status mirror/image_50 2026-02-20T21:38:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_instance=cluster1 2026-02-20T21:38:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:38:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance=cluster1 2026-02-20T21:38:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local loop_instance 2026-02-20T21:38:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:00.980 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sleep 0 2026-02-20T21:38:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -i state 2026-02-20T21:38:00.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ seq 0 3 2026-02-20T21:38:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/image_50 2026-02-20T21:38:00.983 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:0 2026-02-20T21:38:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:38:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:38:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:38:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:38:00.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:0 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:0 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:00.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:00.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:00.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=0 2026-02-20T21:38:00.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=0 2026-02-20T21:38:00.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:00.993 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:00.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/image_50 2026-02-20T21:38:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-20T21:38:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror status : get status for rbd mirror 2026-02-20T21:38:01.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:38:01.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:01.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/image_50 2026-02-20T21:38:01.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1:1 2026-02-20T21:38:01.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ shift 2026-02-20T21:38:01.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:38:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=cluster 2026-02-20T21:38:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval cluster=cluster1 2026-02-20T21:38:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:38:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:38:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:38:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local local_cluster=cluster1:1 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster=cluster1 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local val=cluster1:1 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local cluster_var_name=local_cluster 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local instance_var_name=instance 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ cluster=cluster1 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval local_cluster=cluster1 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ local_cluster=cluster1 2026-02-20T21:38:01.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ eval instance=1 2026-02-20T21:38:01.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ instance=1 2026-02-20T21:38:01.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:01.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:01.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/image_50 2026-02-20T21:38:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ return 0 2026-02-20T21:38:01.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ status_result=' "state": "Replaying"' 2026-02-20T21:38:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo ' "state": "Replaying"' 2026-02-20T21:38:01.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ grep -i Replaying 2026-02-20T21:38:01.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: "state": "Replaying" 2026-02-20T21:38:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=started 2026-02-20T21:38:01.101 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test started = started 2026-02-20T21:38:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror mirror image_50 2026-02-20T21:38:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:38:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:38:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:38:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' journal = journal ']' 2026-02-20T21:38:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror mirror image_50 2026-02-20T21:38:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_cluster=cluster1 2026-02-20T21:38:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:01.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local local_pool=mirror 2026-02-20T21:38:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local remote_pool=mirror 2026-02-20T21:38:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-20T21:38:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-20T21:38:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:38:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:38:01.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:38:01.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_50 2026-02-20T21:38:01.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:01.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:01.306 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:38:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_50 ']' 2026-02-20T21:38:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-02-20T21:38:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-02-20T21:38:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:38:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:38:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:38:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:38:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-02-20T21:38:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:38:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:01.308 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:38:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:38:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:38:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:01.309 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:38:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:38:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:01.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:38:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:38:01.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:01.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-20T21:38:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:38:01.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:38:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_50 2026-02-20T21:38:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:38:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:01.363 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:38:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:38:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:01.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:38:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:38:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:01.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:38:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:01.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:01.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:38:01.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:38:01.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:01.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:01.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-02-20T21:38:01.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:01.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:01.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-02-20T21:38:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:38:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:01.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:01.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:01.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:01.424 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:01.451 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:01.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:01.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:38:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-02-20T21:38:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-02-20T21:38:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:01.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:38:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:01.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:01.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:01.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:01.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:01.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:01.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:01.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:01.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:01.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:01.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23858' '!=' '' 2026-02-20T21:38:01.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ break 2026-02-20T21:38:01.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23858' = '' 2026-02-20T21:38:01.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ last_mirror_pos='object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:01.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:38:01.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:38:01.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:38:01.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_tag=2 2026-02-20T21:38:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-20T21:38:01.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:38:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_tag=1 2026-02-20T21:38:01.490 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:38:01.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:38:01.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:38:01.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_entry=1 2026-02-20T21:38:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-20T21:38:01.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut -d= -f 2 2026-02-20T21:38:01.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_entry=23858 2026-02-20T21:38:01.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 2 = 1 -a 1 -le 23858 2026-02-20T21:38:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:38:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:38:01.494 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.2 2026-02-20T21:38:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_50 2026-02-20T21:38:01.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:38:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_50 ']' 2026-02-20T21:38:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-02-20T21:38:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-02-20T21:38:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:38:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:38:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:38:01.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:01.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:38:01.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:01.699 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-02-20T21:38:01.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:38:01.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:01.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:01.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:01.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:38:01.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:01.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:01.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:01.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:38:01.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:01.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:01.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:01.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:01.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:01.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:01.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:38:01.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:38:01.702 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:01.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-20T21:38:01.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:38:01.752 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:38:01.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:01.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_50 2026-02-20T21:38:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:38:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:38:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:38:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:01.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:38:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:38:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:38:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:01.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:38:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:38:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:01.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:01.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-02-20T21:38:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:01.813 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-02-20T21:38:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:38:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:01.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:01.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:01.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:01.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:01.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:01.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:01.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:38:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-02-20T21:38:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:01.845 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-02-20T21:38:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:38:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:01.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:01.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:01.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:01.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:01.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:01.875 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:01.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:01.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:01.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23858' '!=' 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:01.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:38:01.877 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.4 2026-02-20T21:38:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_50 2026-02-20T21:38:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:02.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:38:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_50 ']' 2026-02-20T21:38:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-02-20T21:38:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-02-20T21:38:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:38:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:38:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:38:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:02.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:02.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:38:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-02-20T21:38:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:38:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:02.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:38:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:38:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:38:02.284 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:38:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:38:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:02.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:02.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:02.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:38:02.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:38:02.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-20T21:38:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:38:02.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:38:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_50 2026-02-20T21:38:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:38:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:38:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:02.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:38:02.340 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:38:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:38:02.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:02.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-02-20T21:38:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:02.398 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-02-20T21:38:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:38:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:02.399 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:02.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:02.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:02.400 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:02.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:02.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:38:02.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-02-20T21:38:02.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-02-20T21:38:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:38:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:02.431 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:02.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:02.458 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:02.459 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:02.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:02.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:02.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:02.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23858' '!=' 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:02.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:38:02.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.8 2026-02-20T21:38:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_50 2026-02-20T21:38:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:38:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_50 ']' 2026-02-20T21:38:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-02-20T21:38:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-02-20T21:38:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:38:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:38:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:38:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:03.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:38:03.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:03.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-02-20T21:38:03.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:38:03.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:03.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:03.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:03.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:38:03.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:03.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:38:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:38:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:38:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:03.269 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:38:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:38:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:38:03.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:03.277 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:03.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:03.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-20T21:38:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:38:03.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:38:03.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_50 2026-02-20T21:38:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:38:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:38:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:03.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:38:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:38:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:38:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:38:03.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:03.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:38:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:38:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:03.335 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:03.336 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:03.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:03.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-02-20T21:38:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-02-20T21:38:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:38:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:03.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:03.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:03.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:03.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:03.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:03.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:03.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:03.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:03.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:03.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:38:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-02-20T21:38:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-02-20T21:38:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:38:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:03.420 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:03.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:03.448 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23858' '!=' 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:38:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1.6 2026-02-20T21:38:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_50 2026-02-20T21:38:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:05.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:38:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_50 ']' 2026-02-20T21:38:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-02-20T21:38:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-02-20T21:38:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:38:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:38:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:38:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:05.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:05.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:38:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-02-20T21:38:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:38:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:38:05.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:38:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:38:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:38:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:05.057 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:38:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:38:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:38:05.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:05.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-20T21:38:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:38:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:38:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_50 2026-02-20T21:38:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:38:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:38:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:38:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:38:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:05.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:05.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:38:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:38:05.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:05.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:05.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-02-20T21:38:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-02-20T21:38:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:05.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:38:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:05.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:05.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:05.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:05.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:05.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:05.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:38:05.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-02-20T21:38:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-02-20T21:38:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:38:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:05.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:05.206 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:05.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:05.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23858' '!=' 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:38:05.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:38:07.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_50 2026-02-20T21:38:07.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:38:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_50 ']' 2026-02-20T21:38:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-02-20T21:38:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-02-20T21:38:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:38:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:07.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:38:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:38:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:07.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:38:07.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:07.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-02-20T21:38:07.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:38:07.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:07.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:07.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:07.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:38:07.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:07.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:38:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:38:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:38:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:07.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:38:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:38:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:38:07.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:07.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:07.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:07.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:07.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-20T21:38:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:38:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:38:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_50 2026-02-20T21:38:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:38:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:38:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:38:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:38:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:38:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:38:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:07.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:38:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:38:07.302 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:07.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:07.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:07.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:07.310 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:07.357 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:07.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-02-20T21:38:07.358 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-02-20T21:38:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:38:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:07.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:07.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:07.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:07.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:07.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:07.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:07.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:07.389 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:07.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:38:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-02-20T21:38:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-02-20T21:38:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:07.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:38:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:07.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:07.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:07.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:07.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:07.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:07.621 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:07.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:07.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:07.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:07.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:07.625 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23858' '!=' 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:38:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:38:09.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_50 2026-02-20T21:38:09.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:09.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:09.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:09.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:38:09.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_50 ']' 2026-02-20T21:38:09.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-02-20T21:38:09.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-02-20T21:38:09.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:38:09.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:09.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:09.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:38:09.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:38:09.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:09.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:09.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:09.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:38:09.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:09.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-02-20T21:38:09.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:38:09.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:09.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:09.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:09.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:38:09.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:09.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:09.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:09.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:38:09.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:09.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:09.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:09.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:38:09.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:09.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:09.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:38:09.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:09.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:09.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:09.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:38:09.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:09.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:09.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:09.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:09.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:09.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:09.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:09.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:38:09.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:38:09.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:09.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:09.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-20T21:38:09.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:38:09.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:38:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_50 2026-02-20T21:38:09.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:38:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:38:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:38:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:09.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:38:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:38:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:38:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:09.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:38:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:38:09.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:09.700 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:09.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-02-20T21:38:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-02-20T21:38:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:09.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:09.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:09.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:38:09.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:09.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:09.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:09.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:09.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:09.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:09.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:09.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:09.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:38:09.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-02-20T21:38:09.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:09.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:09.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:09.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-02-20T21:38:09.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:09.782 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:38:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:09.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:09.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:09.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:09.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:09.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:09.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:09.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=22, tag_tid=1, entry_tid=23858], [object_number=21, tag_tid=1, entry_tid=23857], [object_number=20, tag_tid=1, entry_tid=23856], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:09.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 'object_number=22, tag_tid=1, entry_tid=23858' '!=' 'object_number=22, tag_tid=1, entry_tid=23858' 2026-02-20T21:38:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-20T21:38:09.814 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-20T21:38:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ flush cluster1 mirror image_50 2026-02-20T21:38:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local 'cmd=rbd mirror flush' 2026-02-20T21:38:13.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n image_50 ']' 2026-02-20T21:38:13.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmd='rbd mirror flush mirror/image_50' 2026-02-20T21:38:13.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/image_50 2026-02-20T21:38:13.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_instance=cluster1 2026-02-20T21:38:13.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:13.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:13.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance=cluster1 2026-02-20T21:38:13.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loop_instance 2026-02-20T21:38:13.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-20T21:38:13.817 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-20T21:38:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:38:13.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/image_50 2026-02-20T21:38:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:38:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-20T21:38:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:0 2026-02-20T21:38:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:38:13.821 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:13.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:13.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:13.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:38:13.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:13.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-20T21:38:13.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:0 2026-02-20T21:38:13.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:13.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:13.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-20T21:38:13.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:0 2026-02-20T21:38:13.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:13.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:13.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:13.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:38:13.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-20T21:38:13.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:13.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:13.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=0 2026-02-20T21:38:13.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=0 2026-02-20T21:38:13.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-20T21:38:13.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:13.832 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:13.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-20T21:38:13.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:Invalid command: unused arguments: ['mirror/image_50'] 2026-02-20T21:38:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd mirror flush : flush rbd mirror 2026-02-20T21:38:13.876 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:admin_socket: invalid command 2026-02-20T21:38:13.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:38:13.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/image_50 2026-02-20T21:38:13.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:38:13.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:38:13.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:38:13.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-20T21:38:13.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1:1 2026-02-20T21:38:13.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:38:13.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:38:13.880 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:38:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=1 2026-02-20T21:38:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:38:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=1 2026-02-20T21:38:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-20T21:38:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local local_cluster=cluster1:1 2026-02-20T21:38:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:38:13.881 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance 2026-02-20T21:38:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-20T21:38:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local val=cluster1:1 2026-02-20T21:38:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster_var_name=local_cluster 2026-02-20T21:38:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local instance_var_name=instance 2026-02-20T21:38:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:38:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=1 2026-02-20T21:38:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-20T21:38:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval local_cluster=cluster1 2026-02-20T21:38:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ local_cluster=cluster1 2026-02-20T21:38:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ eval instance=1 2026-02-20T21:38:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ instance=1 2026-02-20T21:38:13.882 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-20T21:38:13.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:13.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:13.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-20T21:38:13.890 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/image_50 2026-02-20T21:38:13.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_master_journal_position cluster2 mirror image_50 2026-02-20T21:38:13.941 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '' 2026-02-20T21:38:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local id_regexp= 2026-02-20T21:38:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:13.942 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:13.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:13.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:13.943 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:13.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:13.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:13.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:13.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:13.971 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:13.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:13.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ master_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:38:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_mirror_journal_position cluster2 mirror image_50 2026-02-20T21:38:13.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_journal_position cluster2 mirror image_50 '..*' 2026-02-20T21:38:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:38:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:38:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:38:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local 'id_regexp=..*' 2026-02-20T21:38:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ mkfname cluster2-mirror-image_50.status 2026-02-20T21:38:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ echo cluster2-mirror-image_50.status 2026-02-20T21:38:13.975 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-20T21:38:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local status_log=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster2 journal status --image mirror/image_50 2026-02-20T21:38:13.976 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ tee /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:minimum_set: 5 2026-02-20T21:38:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:active_set: 5 2026-02-20T21:38:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:registered clients: 2026-02-20T21:38:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:14.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: [id=8b506631-c182-41c2-9387-8476b842b351, commit_position=[positions=[[object_number=21, tag_tid=2, entry_tid=1], [object_number=20, tag_tid=2, entry_tid=0], [object_number=22, tag_tid=1, entry_tid=23858], [object_number=23, tag_tid=1, entry_tid=23855]]], state=connected] 2026-02-20T21:38:14.005 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.status 2026-02-20T21:38:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ mirror_pos='object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:38:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 'object_number=21, tag_tid=2, entry_tid=1' -a 'object_number=21, tag_tid=2, entry_tid=1' = 'object_number=21, tag_tid=2, entry_tid=1' 2026-02-20T21:38:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_snap_present cluster1 mirror image_50 snap 2026-02-20T21:38:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:38:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:38:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:38:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-20T21:38:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:38:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 info mirror/image_50@snap 2026-02-20T21:38:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_50': 2026-02-20T21:38:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 128 MiB in 32 objects 2026-02-20T21:38:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:38:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 1 2026-02-20T21:38:15.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 11b3af8129b4 2026-02-20T21:38:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.11b3af8129b4 2026-02-20T21:38:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:38:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:38:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:38:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:38:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:26:35 2026 2026-02-20T21:38:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:26:35 2026 2026-02-20T21:38:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:28:36 2026 2026-02-20T21:38:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: protected: False 2026-02-20T21:38:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 11b3af8129b4 2026-02-20T21:38:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:38:15.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:38:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 976dfaff-5d00-428a-b058-f0f372cd6d5d 2026-02-20T21:38:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: false 2026-02-20T21:38:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ compare_image_snaps mirror image_50 snap 2026-02-20T21:38:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:38:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap_name=snap 2026-02-20T21:38:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local ret=0 2026-02-20T21:38:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local rmt_export=/tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.export 2026-02-20T21:38:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local loc_export=/tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_50.export 2026-02-20T21:38:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_50.export 2026-02-20T21:38:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 -p mirror export image_50@snap /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.export 2026-02-20T21:38:15.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:38:15.167 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster1 -p mirror export image_50@snap /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_50.export 2026-02-20T21:38:15.200 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:38:15.198+0000 7fc9bc805640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2026-02-20T21:38:15.200 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:38:15.199+0000 7fc9be008640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2026-02-20T21:38:15.201 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:38:15.199+0000 7fc9be809640 -1 bdev(0x5576e00fa000 /var/lib/ceph/osd/cluster1-2/block) aio_submit retries 1 2026-02-20T21:38:15.296 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-20T21:38:15.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cmp /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_50.export 2026-02-20T21:38:15.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -f /tmp/tmp.IQNOxgiLiE/cluster2-mirror-image_50.export /tmp/tmp.IQNOxgiLiE/cluster1-mirror-image_50.export 2026-02-20T21:38:15.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:15.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: delete many images' 2026-02-20T21:38:15.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.IQNOxgiLiE/rbd-mirror.test.log 2026-02-20T21:38:15.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-20T21:38:15.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-20 21:38:15 TEST: delete many images 2026-02-20T21:38:15.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-20 21:38:15 TEST: delete many images 2026-02-20T21:38:15.387 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 50 2026-02-20T21:38:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_1 2026-02-20T21:38:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_1 snap 2026-02-20T21:38:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:38:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:15.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_1@snap 2026-02-20T21:38:15.978 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_1 2026-02-20T21:38:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:15.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:38:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:15.986 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_1 2026-02-20T21:38:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:38:15.987 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-20T21:38:16.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-20T21:38:16.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_2 2026-02-20T21:38:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_2 snap 2026-02-20T21:38:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:38:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_2@snap 2026-02-20T21:38:16.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_2 2026-02-20T21:38:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:38:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:16.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:16.995 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_2 2026-02-20T21:38:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:38:16.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-20T21:38:17.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-20T21:38:17.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_3 2026-02-20T21:38:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_3 snap 2026-02-20T21:38:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:38:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:17.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_3@snap 2026-02-20T21:38:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:17.997 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_3 2026-02-20T21:38:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:38:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:17.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_3 2026-02-20T21:38:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:38:17.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-20T21:38:18.027 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-20T21:38:18.124 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_4 2026-02-20T21:38:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_4 snap 2026-02-20T21:38:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:38:18.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:18.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_4@snap 2026-02-20T21:38:18.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_4 2026-02-20T21:38:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:38:19.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_4 2026-02-20T21:38:19.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:38:19.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-20T21:38:19.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-20T21:38:19.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_5 2026-02-20T21:38:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_5 snap 2026-02-20T21:38:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:38:19.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:19.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_5@snap 2026-02-20T21:38:19.992 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_5 2026-02-20T21:38:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:38:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_5 2026-02-20T21:38:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:38:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-20T21:38:20.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-20T21:38:20.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:20.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_6 2026-02-20T21:38:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_6 snap 2026-02-20T21:38:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:38:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:20.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_6@snap 2026-02-20T21:38:20.994 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_6 2026-02-20T21:38:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:38:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:21.002 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_6 2026-02-20T21:38:21.003 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:38:21.004 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-20T21:38:21.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-20T21:38:21.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:21.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:21.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:21.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_7 2026-02-20T21:38:21.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_7 snap 2026-02-20T21:38:21.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:21.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:21.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:38:21.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:21.142 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_7@snap 2026-02-20T21:38:22.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_7 2026-02-20T21:38:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:38:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:22.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_7 2026-02-20T21:38:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:38:22.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-20T21:38:22.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-20T21:38:22.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:22.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:22.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:22.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_8 2026-02-20T21:38:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_8 snap 2026-02-20T21:38:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:38:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:22.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_8@snap 2026-02-20T21:38:23.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_8 2026-02-20T21:38:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:38:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_8 2026-02-20T21:38:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:38:23.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-20T21:38:23.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-20T21:38:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:23.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:23.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:23.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_9 2026-02-20T21:38:23.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_9 snap 2026-02-20T21:38:23.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:23.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:38:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_9@snap 2026-02-20T21:38:24.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_9 2026-02-20T21:38:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:38:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:24.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_9 2026-02-20T21:38:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:38:24.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-20T21:38:24.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-20T21:38:24.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:24.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:24.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:24.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_10 2026-02-20T21:38:24.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_10 snap 2026-02-20T21:38:24.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:24.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:24.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:38:24.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:24.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_10@snap 2026-02-20T21:38:25.036 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:25.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_10 2026-02-20T21:38:25.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:25.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:25.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:38:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:25.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_10 2026-02-20T21:38:25.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:25.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:25.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:38:25.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-20T21:38:25.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-20T21:38:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_11 2026-02-20T21:38:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_11 snap 2026-02-20T21:38:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:38:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:25.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_11@snap 2026-02-20T21:38:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_11 2026-02-20T21:38:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:38:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_11 2026-02-20T21:38:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:38:26.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_11 2026-02-20T21:38:26.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_11 2026-02-20T21:38:26.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:26.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:26.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:26.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_12 2026-02-20T21:38:26.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_12 snap 2026-02-20T21:38:26.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:26.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:38:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:26.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_12@snap 2026-02-20T21:38:27.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_12 2026-02-20T21:38:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:38:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_12 2026-02-20T21:38:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:27.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:27.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:38:27.054 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_12 2026-02-20T21:38:27.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_12 2026-02-20T21:38:27.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_13 2026-02-20T21:38:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_13 snap 2026-02-20T21:38:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:38:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_13@snap 2026-02-20T21:38:28.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_13 2026-02-20T21:38:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:38:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:28.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_13 2026-02-20T21:38:28.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:28.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:28.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:38:28.052 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_13 2026-02-20T21:38:28.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_13 2026-02-20T21:38:28.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:28.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:28.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:28.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_14 2026-02-20T21:38:28.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_14 snap 2026-02-20T21:38:28.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:28.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:28.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:38:28.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:28.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_14@snap 2026-02-20T21:38:29.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_14 2026-02-20T21:38:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:38:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_14 2026-02-20T21:38:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:38:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_14 2026-02-20T21:38:29.087 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_14 2026-02-20T21:38:29.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_15 2026-02-20T21:38:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_15 snap 2026-02-20T21:38:29.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:38:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:29.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_15@snap 2026-02-20T21:38:30.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:30.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_15 2026-02-20T21:38:30.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:30.059 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:30.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:38:30.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:30.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:30.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_15 2026-02-20T21:38:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:38:30.061 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_15 2026-02-20T21:38:30.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_15 2026-02-20T21:38:30.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:30.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:30.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:30.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_16 2026-02-20T21:38:30.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_16 snap 2026-02-20T21:38:30.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:30.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:30.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:38:30.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:30.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_16@snap 2026-02-20T21:38:31.032 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_16 2026-02-20T21:38:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:38:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_16 2026-02-20T21:38:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:38:31.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_16 2026-02-20T21:38:31.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_16 2026-02-20T21:38:31.083 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:38:31.081+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:38:31.083 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:38:31.081+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b860 2026-02-20T21:38:31.084 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:38:31.081+0000 7fd67aee6640 -1 reset not still connected to 0x55daead272b0 2026-02-20T21:38:31.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:31.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:31.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_17 2026-02-20T21:38:31.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_17 snap 2026-02-20T21:38:31.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:31.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:31.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:38:31.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:31.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_17@snap 2026-02-20T21:38:32.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_17 2026-02-20T21:38:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:38:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:32.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_17 2026-02-20T21:38:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:32.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:32.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:38:32.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_17 2026-02-20T21:38:32.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_17 2026-02-20T21:38:32.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:32.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_18 2026-02-20T21:38:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_18 snap 2026-02-20T21:38:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:38:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_18@snap 2026-02-20T21:38:33.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_18 2026-02-20T21:38:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:38:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:33.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:33.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_18 2026-02-20T21:38:33.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:33.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:33.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:38:33.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_18 2026-02-20T21:38:33.102 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_18 2026-02-20T21:38:33.198 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:33.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_19 2026-02-20T21:38:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_19 snap 2026-02-20T21:38:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:38:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:33.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_19@snap 2026-02-20T21:38:34.063 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_19 2026-02-20T21:38:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:38:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:34.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_19 2026-02-20T21:38:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:38:34.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_19 2026-02-20T21:38:34.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_19 2026-02-20T21:38:34.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_20 2026-02-20T21:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_20 snap 2026-02-20T21:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_20@snap 2026-02-20T21:38:35.062 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:35.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_20 2026-02-20T21:38:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:38:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_20 2026-02-20T21:38:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:38:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_20 2026-02-20T21:38:35.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_20 2026-02-20T21:38:35.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_21 2026-02-20T21:38:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_21 snap 2026-02-20T21:38:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:38:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:35.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_21@snap 2026-02-20T21:38:36.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:36.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_21 2026-02-20T21:38:36.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:36.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:36.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:38:36.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:36.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_21 2026-02-20T21:38:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:38:36.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_21 2026-02-20T21:38:36.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_21 2026-02-20T21:38:36.191 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_22 2026-02-20T21:38:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_22 snap 2026-02-20T21:38:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:38:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:36.196 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_22@snap 2026-02-20T21:38:37.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_22 2026-02-20T21:38:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:37.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:38:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:37.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_22 2026-02-20T21:38:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:38:37.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_22 2026-02-20T21:38:37.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_22 2026-02-20T21:38:37.207 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_23 2026-02-20T21:38:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_23 snap 2026-02-20T21:38:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:38:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_23@snap 2026-02-20T21:38:38.070 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:38.077 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_23 2026-02-20T21:38:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:38:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:38.078 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_23 2026-02-20T21:38:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:38:38.079 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_23 2026-02-20T21:38:38.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_23 2026-02-20T21:38:38.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:38.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:38.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:38.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_24 2026-02-20T21:38:38.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_24 snap 2026-02-20T21:38:38.204 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:38.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:38.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:38:38.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:38.205 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_24@snap 2026-02-20T21:38:39.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_24 2026-02-20T21:38:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:39.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:38:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:39.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_24 2026-02-20T21:38:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:38:39.083 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_24 2026-02-20T21:38:39.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_24 2026-02-20T21:38:39.209 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:39.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_25 2026-02-20T21:38:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_25 snap 2026-02-20T21:38:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:38:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:39.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_25@snap 2026-02-20T21:38:40.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_25 2026-02-20T21:38:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:38:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:40.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_25 2026-02-20T21:38:40.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:40.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:40.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:38:40.085 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_25 2026-02-20T21:38:40.114 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_25 2026-02-20T21:38:40.210 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:40.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:40.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:40.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_26 2026-02-20T21:38:40.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_26 snap 2026-02-20T21:38:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:38:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:40.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_26@snap 2026-02-20T21:38:41.038 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_26 2026-02-20T21:38:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:38:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_26 2026-02-20T21:38:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:38:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_26 2026-02-20T21:38:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_26 2026-02-20T21:38:41.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:41.385 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:41.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:41.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_27 2026-02-20T21:38:41.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_27 snap 2026-02-20T21:38:41.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:41.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:41.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:38:41.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:41.386 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_27@snap 2026-02-20T21:38:42.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:42.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_27 2026-02-20T21:38:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:38:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:42.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_27 2026-02-20T21:38:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:38:42.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_27 2026-02-20T21:38:42.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_27 2026-02-20T21:38:42.220 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_28 2026-02-20T21:38:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_28 snap 2026-02-20T21:38:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:38:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:42.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_28@snap 2026-02-20T21:38:43.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:43.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_28 2026-02-20T21:38:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:38:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:43.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:43.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_28 2026-02-20T21:38:43.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:43.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:43.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:38:43.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_28 2026-02-20T21:38:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_28 2026-02-20T21:38:43.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:43.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_29 2026-02-20T21:38:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_29 snap 2026-02-20T21:38:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:38:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_29@snap 2026-02-20T21:38:44.002 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:38:43.999+0000 7fd67aee6640 -1 reset not still connected to 0x55daafd1ea90 2026-02-20T21:38:44.002 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:38:44.000+0000 7fd67aee6640 -1 reset not still connected to 0x55dab34409c0 2026-02-20T21:38:44.002 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:38:44.000+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b380 2026-02-20T21:38:44.003 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:38:44.001+0000 7fd67aee6640 -1 reset not still connected to 0x55dadc92b860 2026-02-20T21:38:44.003 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:38:44.001+0000 7fd67aee6640 -1 reset not still connected to 0x55daead260d0 2026-02-20T21:38:44.003 INFO:tasks.ceph.cluster1.osd.0.trial069.stderr:2026-02-20T21:38:44.001+0000 7fd67aee6640 -1 reset not still connected to 0x55daead272b0 2026-02-20T21:38:44.086 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:44.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_29 2026-02-20T21:38:44.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:44.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:44.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:38:44.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:44.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:44.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_29 2026-02-20T21:38:44.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:44.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:44.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:38:44.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_29 2026-02-20T21:38:44.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_29 2026-02-20T21:38:44.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_30 2026-02-20T21:38:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_30 snap 2026-02-20T21:38:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:38:44.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:44.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_30@snap 2026-02-20T21:38:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_30 2026-02-20T21:38:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:38:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:45.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_30 2026-02-20T21:38:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:45.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:38:45.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_30 2026-02-20T21:38:45.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_30 2026-02-20T21:38:45.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:45.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:45.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:45.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_31 2026-02-20T21:38:45.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_31 snap 2026-02-20T21:38:45.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:45.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:38:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:45.235 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_31@snap 2026-02-20T21:38:46.090 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:46.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_31 2026-02-20T21:38:46.098 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_31 2026-02-20T21:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_31 2026-02-20T21:38:46.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_31 2026-02-20T21:38:46.221 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_32 2026-02-20T21:38:46.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_32 snap 2026-02-20T21:38:46.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:46.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:46.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:38:46.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:46.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_32@snap 2026-02-20T21:38:47.094 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_32 2026-02-20T21:38:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:38:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:47.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_32 2026-02-20T21:38:47.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:47.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:47.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:38:47.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_32 2026-02-20T21:38:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_32 2026-02-20T21:38:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:47.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:47.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:47.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_33 2026-02-20T21:38:47.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_33 snap 2026-02-20T21:38:47.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:47.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:47.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:38:47.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:47.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_33@snap 2026-02-20T21:38:48.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_33 2026-02-20T21:38:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:38:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:48.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_33 2026-02-20T21:38:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:38:48.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_33 2026-02-20T21:38:48.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_33 2026-02-20T21:38:48.227 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_34 2026-02-20T21:38:48.231 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_34 snap 2026-02-20T21:38:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:38:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:48.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_34@snap 2026-02-20T21:38:49.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_34 2026-02-20T21:38:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:49.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:38:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:49.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_34 2026-02-20T21:38:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:38:49.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_34 2026-02-20T21:38:49.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_34 2026-02-20T21:38:49.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_35 2026-02-20T21:38:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_35 snap 2026-02-20T21:38:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:49.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:38:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:49.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_35@snap 2026-02-20T21:38:50.097 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_35 2026-02-20T21:38:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:38:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:50.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_35 2026-02-20T21:38:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:50.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:38:50.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_35 2026-02-20T21:38:50.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_35 2026-02-20T21:38:50.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:50.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:50.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:50.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_36 2026-02-20T21:38:50.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_36 snap 2026-02-20T21:38:50.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:50.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:50.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:38:50.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:50.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_36@snap 2026-02-20T21:38:51.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:51.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_36 2026-02-20T21:38:51.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:51.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:51.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:38:51.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:51.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:51.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_36 2026-02-20T21:38:51.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:51.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:51.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:38:51.047 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_36 2026-02-20T21:38:51.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_36 2026-02-20T21:38:51.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:51.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:51.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:51.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_37 2026-02-20T21:38:51.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_37 snap 2026-02-20T21:38:51.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:51.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:51.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:38:51.174 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:51.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_37@snap 2026-02-20T21:38:52.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:52.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_37 2026-02-20T21:38:52.115 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:38:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:52.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_37 2026-02-20T21:38:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:38:52.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_37 2026-02-20T21:38:52.138 INFO:tasks.ceph.cluster1.osd.2.trial069.stderr:2026-02-20T21:38:52.136+0000 7fc9d03ed640 -1 reset not still connected to 0x55771baa1040 2026-02-20T21:38:52.145 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_37 2026-02-20T21:38:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:52.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_38 2026-02-20T21:38:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_38 snap 2026-02-20T21:38:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:38:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:52.245 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_38@snap 2026-02-20T21:38:53.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:53.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_38 2026-02-20T21:38:53.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:53.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:53.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:38:53.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:53.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:53.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_38 2026-02-20T21:38:53.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:53.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:53.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:38:53.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_38 2026-02-20T21:38:53.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_38 2026-02-20T21:38:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_39 2026-02-20T21:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_39 snap 2026-02-20T21:38:53.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:38:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:53.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_39@snap 2026-02-20T21:38:54.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:54.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_39 2026-02-20T21:38:54.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:54.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:54.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:38:54.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:54.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_39 2026-02-20T21:38:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:38:54.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_39 2026-02-20T21:38:54.147 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_39 2026-02-20T21:38:54.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:54.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_40 2026-02-20T21:38:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_40 snap 2026-02-20T21:38:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:38:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:54.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_40@snap 2026-02-20T21:38:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:55.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_40 2026-02-20T21:38:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:38:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:55.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_40 2026-02-20T21:38:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:38:55.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_40 2026-02-20T21:38:55.146 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_40 2026-02-20T21:38:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_41 2026-02-20T21:38:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_41 snap 2026-02-20T21:38:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:38:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:55.247 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_41@snap 2026-02-20T21:38:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_41 2026-02-20T21:38:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:38:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:56.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_41 2026-02-20T21:38:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:56.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:38:56.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_41 2026-02-20T21:38:56.148 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_41 2026-02-20T21:38:56.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_42 2026-02-20T21:38:56.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_42 snap 2026-02-20T21:38:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:38:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:56.241 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_42@snap 2026-02-20T21:38:57.098 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.0 2026-02-20T21:38:57.099 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Sent signal 15 2026-02-20T21:38:57.099 INFO:tasks.rbd_mirror_thrash:waiting for 2.0113202960721943 secs before reviving daemons 2026-02-20T21:38:57.100 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:38:57.098+0000 7f40bc0a8640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 119424) UID: 1000 2026-02-20T21:38:57.113 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:57.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_42 2026-02-20T21:38:57.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:57.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:57.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:38:57.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:57.122 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_42 2026-02-20T21:38:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:38:57.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_42 2026-02-20T21:38:57.154 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_42 2026-02-20T21:38:57.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_43 2026-02-20T21:38:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_43 snap 2026-02-20T21:38:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:38:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:57.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_43@snap 2026-02-20T21:38:58.117 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:58.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_43 2026-02-20T21:38:58.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:58.125 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:58.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:38:58.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:58.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:58.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_43 2026-02-20T21:38:58.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:58.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:58.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:38:58.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_43 2026-02-20T21:38:58.155 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_43 2026-02-20T21:38:58.248 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:58.252 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_44 2026-02-20T21:38:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_44 snap 2026-02-20T21:38:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:38:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:58.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_44@snap 2026-02-20T21:38:59.111 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.0 2026-02-20T21:38:59.111 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:38:59.112 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.0 2026-02-20T21:38:59.112 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Restarting daemon 2026-02-20T21:38:59.112 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 2026-02-20T21:38:59.114 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Started 2026-02-20T21:38:59.115 INFO:tasks.rbd_mirror_thrash:waiting for 108.45645735142563 secs before thrashing 2026-02-20T21:38:59.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:38:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_44 2026-02-20T21:38:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:59.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:38:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:38:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:38:59.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_44 2026-02-20T21:38:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:38:59.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_44 2026-02-20T21:38:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_44 2026-02-20T21:38:59.202 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-20T21:38:59.199+0000 7fdf668cb640 -1 reset not still connected to 0x55a651304410 2026-02-20T21:38:59.202 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-20T21:38:59.199+0000 7fdf668cb640 -1 reset not still connected to 0x55a656376000 2026-02-20T21:38:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:38:59.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:38:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:38:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_45 2026-02-20T21:38:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_45 snap 2026-02-20T21:38:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:38:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:38:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:38:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:38:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_45@snap 2026-02-20T21:39:00.119 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:39:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_45 2026-02-20T21:39:00.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:39:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:39:00.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:39:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_45 2026-02-20T21:39:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:39:00.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_45 2026-02-20T21:39:00.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_45 2026-02-20T21:39:00.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:39:00.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:00.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:00.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_46 2026-02-20T21:39:00.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_46 snap 2026-02-20T21:39:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:39:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:39:00.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_46@snap 2026-02-20T21:39:01.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:39:01.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_46 2026-02-20T21:39:01.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:01.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:01.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:39:01.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:39:01.044 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:39:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_46 2026-02-20T21:39:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:39:01.045 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_46 2026-02-20T21:39:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_46 2026-02-20T21:39:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:39:01.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:01.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:01.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_47 2026-02-20T21:39:01.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_47 snap 2026-02-20T21:39:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:39:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:39:01.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_47@snap 2026-02-20T21:39:02.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_47 2026-02-20T21:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:02.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:39:02.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_47 2026-02-20T21:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:39:02.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_47 2026-02-20T21:39:02.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_47 2026-02-20T21:39:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:39:02.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:02.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:02.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_48 2026-02-20T21:39:02.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_48 snap 2026-02-20T21:39:02.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:02.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:39:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:39:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_48@snap 2026-02-20T21:39:03.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:39:03.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_48 2026-02-20T21:39:03.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:03.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:03.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:39:03.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:39:03.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:39:03.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_48 2026-02-20T21:39:03.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:03.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:03.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:39:03.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_48 2026-02-20T21:39:03.162 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_48 2026-02-20T21:39:03.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:39:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_49 2026-02-20T21:39:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_49 snap 2026-02-20T21:39:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:39:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:39:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_49@snap 2026-02-20T21:39:04.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:39:04.137 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_49 2026-02-20T21:39:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:39:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:39:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:39:04.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_49 2026-02-20T21:39:04.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:04.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:04.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:39:04.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_49 2026-02-20T21:39:04.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_49 2026-02-20T21:39:04.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:39:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:04.271 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_50 2026-02-20T21:39:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_snapshot cluster2 mirror image_50 snap 2026-02-20T21:39:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:39:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local snap=snap 2026-02-20T21:39:04.272 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap rm mirror/image_50@snap 2026-02-20T21:39:05.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing snap: 100% complete...done. 2026-02-20T21:39:05.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_50 2026-02-20T21:39:05.138 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:39:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:39:05.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:39:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_50 2026-02-20T21:39:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:39:05.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_50 2026-02-20T21:39:05.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_50 2026-02-20T21:39:05.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-20T21:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: image deletions should propagate' 2026-02-20T21:39:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.IQNOxgiLiE/rbd-mirror.test.log 2026-02-20T21:39:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-20T21:39:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-20 21:39:05 TEST: image deletions should propagate 2026-02-20T21:39:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-20 21:39:05 TEST: image deletions should propagate 2026-02-20T21:39:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_pool_images cluster1 mirror 0 2026-02-20T21:39:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_count=0 2026-02-20T21:39:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local count 2026-02-20T21:39:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local last_count=0 2026-02-20T21:39:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ true 2026-02-20T21:39:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 40 2026-02-20T21:39:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-20T21:39:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 1 -ne 1 2026-02-20T21:39:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-20T21:39:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'images: ' 2026-02-20T21:39:05.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 2 2026-02-20T21:39:05.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T21:39:05.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ count=0 2026-02-20T21:39:05.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-20T21:39:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_pool_healthy cluster1 mirror 0 2026-02-20T21:39:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state 2026-02-20T21:39:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 40 2026-02-20T21:39:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in `seq 1 40` 2026-02-20T21:39:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 1 -ne 1 2026-02-20T21:39:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster cluster1 -p mirror mirror pool status 2026-02-20T21:39:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep 'image health:' 2026-02-20T21:39:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cut '-d ' -f 3 2026-02-20T21:39:05.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: failed to query services: (13) Permission denied 2026-02-20T21:39:05.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ state=OK 2026-02-20T21:39:05.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test OK = ERROR 2026-02-20T21:39:05.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test OK = OK 2026-02-20T21:39:05.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:05.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 50 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_1 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_1 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:39:05.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-20T21:39:05.531 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-20T21:39:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-02-20T21:39:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_1 2026-02-20T21:39:05.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:39:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-20T21:39:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-20T21:39:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_2 2026-02-20T21:39:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T21:39:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T21:39:05.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:05.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:05.705 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_2 deleted '' 2026-02-20T21:39:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:39:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T21:39:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:39:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T21:39:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:05.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T21:39:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_3 2026-02-20T21:39:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-02-20T21:39:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:05.833 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:39:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_3 2026-02-20T21:39:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:39:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-20T21:39:05.834 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:05.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-20T21:39:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:05.857 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-02-20T21:39:05.958 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_3 2026-02-20T21:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-20T21:39:05.959 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:05.981 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_4 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_4 2026-02-20T21:39:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:39:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-20T21:39:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.007 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-20T21:39:06.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:06.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:06.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:06.110 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-02-20T21:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_4 2026-02-20T21:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:39:06.111 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-20T21:39:06.112 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-20T21:39:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_5 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_5 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-20T21:39:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-20T21:39:06.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:06.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:06.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:06.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-02-20T21:39:06.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:06.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:06.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:39:06.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:06.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_5 2026-02-20T21:39:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:39:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-20T21:39:06.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-20T21:39:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:06.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_6 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_6 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-20T21:39:06.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-20T21:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:06.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:06.413 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_6 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-20T21:39:06.414 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-20T21:39:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_7 2026-02-20T21:39:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-02-20T21:39:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:39:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:06.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:06.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:06.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_7 2026-02-20T21:39:06.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:06.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:06.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:39:06.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-20T21:39:06.439 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.460 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-20T21:39:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:06.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-02-20T21:39:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:06.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:39:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_7 2026-02-20T21:39:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:39:06.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-20T21:39:06.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-20T21:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:06.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_8 2026-02-20T21:39:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-02-20T21:39:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:06.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:39:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_8 2026-02-20T21:39:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:39:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-20T21:39:06.593 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-20T21:39:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-02-20T21:39:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:39:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_8 2026-02-20T21:39:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:06.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:39:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-20T21:39:06.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-20T21:39:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_9 2026-02-20T21:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-02-20T21:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_9 2026-02-20T21:39:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:39:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-20T21:39:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-20T21:39:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:06.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:06.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-02-20T21:39:06.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:06.870 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:39:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_9 2026-02-20T21:39:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:39:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-20T21:39:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-20T21:39:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_10 2026-02-20T21:39:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-02-20T21:39:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:39:06.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:06.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:06.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:06.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:06.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_10 2026-02-20T21:39:06.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:06.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:06.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-20T21:39:06.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-20T21:39:06.898 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:06.920 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-20T21:39:06.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:06.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:06.922 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-02-20T21:39:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:39:07.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_10 2026-02-20T21:39:07.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-20T21:39:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-20T21:39:07.026 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.046 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-20T21:39:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:07.049 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_11 2026-02-20T21:39:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_11 deleted 2026-02-20T21:39:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:39:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:07.050 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_11 2026-02-20T21:39:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:39:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-02-20T21:39:07.051 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-02-20T21:39:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_11 deleted '' 2026-02-20T21:39:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_11 2026-02-20T21:39:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_11 2026-02-20T21:39:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.176 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_11 2026-02-20T21:39:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_11 2026-02-20T21:39:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.197 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_11: (2) No such file or directory 2026-02-20T21:39:07.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:07.200 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:07.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:07.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:07.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_12 2026-02-20T21:39:07.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_12 deleted 2026-02-20T21:39:07.201 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:39:07.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:07.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:07.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.202 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_12 2026-02-20T21:39:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:39:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-02-20T21:39:07.203 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.223 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-02-20T21:39:07.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:07.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:07.226 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_12 deleted '' 2026-02-20T21:39:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_12 2026-02-20T21:39:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_12 2026-02-20T21:39:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_12 2026-02-20T21:39:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_12 2026-02-20T21:39:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_12: (2) No such file or directory 2026-02-20T21:39:07.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:07.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:07.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:07.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:07.352 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_13 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_13 deleted 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_13 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-02-20T21:39:07.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-02-20T21:39:07.376 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:07.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:07.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:07.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_13 deleted '' 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_13 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_13 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_13 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_13 2026-02-20T21:39:07.479 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_13: (2) No such file or directory 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_14 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_14 deleted 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_14 2026-02-20T21:39:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:39:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-02-20T21:39:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-02-20T21:39:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:07.526 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:07.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_14 deleted '' 2026-02-20T21:39:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_14 2026-02-20T21:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_14 2026-02-20T21:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_14 2026-02-20T21:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_14 2026-02-20T21:39:07.628 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.650 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_14: (2) No such file or directory 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_15 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_15 deleted 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:07.652 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_15 2026-02-20T21:39:07.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:39:07.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-02-20T21:39:07.653 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-02-20T21:39:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_15 deleted '' 2026-02-20T21:39:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_15 2026-02-20T21:39:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_15 2026-02-20T21:39:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_15 2026-02-20T21:39:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_15 2026-02-20T21:39:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.802 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_15: (2) No such file or directory 2026-02-20T21:39:07.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:07.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:07.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:07.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:07.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_16 2026-02-20T21:39:07.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_16 deleted 2026-02-20T21:39:07.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:39:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_16 2026-02-20T21:39:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:39:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-02-20T21:39:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-02-20T21:39:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_16 deleted '' 2026-02-20T21:39:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_16 2026-02-20T21:39:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_16 2026-02-20T21:39:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_16 2026-02-20T21:39:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_16 2026-02-20T21:39:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_16: (2) No such file or directory 2026-02-20T21:39:07.954 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_17 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_17 deleted 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_17 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-02-20T21:39:07.955 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:07.977 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-02-20T21:39:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:07.979 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_17 deleted '' 2026-02-20T21:39:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_17 2026-02-20T21:39:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_17 2026-02-20T21:39:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_17 2026-02-20T21:39:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_17 2026-02-20T21:39:08.082 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:08.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_17: (2) No such file or directory 2026-02-20T21:39:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_18 2026-02-20T21:39:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_18 deleted 2026-02-20T21:39:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:39:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_18 2026-02-20T21:39:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:39:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-02-20T21:39:08.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:08.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-02-20T21:39:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_18 deleted '' 2026-02-20T21:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_18 2026-02-20T21:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.232 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_18 2026-02-20T21:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_18 2026-02-20T21:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_18 2026-02-20T21:39:08.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:08.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_18: (2) No such file or directory 2026-02-20T21:39:08.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:08.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_19 2026-02-20T21:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_19 deleted 2026-02-20T21:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:08.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_19 2026-02-20T21:39:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:39:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-02-20T21:39:08.259 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:08.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-02-20T21:39:08.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_19 deleted '' 2026-02-20T21:39:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_19 2026-02-20T21:39:08.382 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_19 2026-02-20T21:39:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_19 2026-02-20T21:39:08.383 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_19 2026-02-20T21:39:08.384 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_19: (2) No such file or directory 2026-02-20T21:39:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:08.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_20 2026-02-20T21:39:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_20 deleted 2026-02-20T21:39:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:39:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_20 2026-02-20T21:39:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.411 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:39:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-02-20T21:39:08.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:08.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-02-20T21:39:08.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:08.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:08.435 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_20 deleted '' 2026-02-20T21:39:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_20 2026-02-20T21:39:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:08.537 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_20 2026-02-20T21:39:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_20 2026-02-20T21:39:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_20 2026-02-20T21:39:08.538 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:08.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_20: (2) No such file or directory 2026-02-20T21:39:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_21 2026-02-20T21:39:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_21 deleted 2026-02-20T21:39:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:39:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_21 2026-02-20T21:39:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:39:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-02-20T21:39:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:08.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-02-20T21:39:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:08.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_21 deleted '' 2026-02-20T21:39:08.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_21 2026-02-20T21:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_21 2026-02-20T21:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_21 2026-02-20T21:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_21 2026-02-20T21:39:08.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:08.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_21: (2) No such file or directory 2026-02-20T21:39:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:08.712 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_22 2026-02-20T21:39:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_22 deleted 2026-02-20T21:39:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:39:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:08.713 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_22 2026-02-20T21:39:08.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:39:08.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-02-20T21:39:08.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:08.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-02-20T21:39:08.736 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:08.737 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_22 deleted '' 2026-02-20T21:39:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_22 2026-02-20T21:39:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:08.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_22 2026-02-20T21:39:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_22 2026-02-20T21:39:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_22 2026-02-20T21:39:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:08.861 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_22: (2) No such file or directory 2026-02-20T21:39:08.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:08.864 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:08.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:08.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:08.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:08.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_23 2026-02-20T21:39:08.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_23 deleted 2026-02-20T21:39:08.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:39:08.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:08.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:08.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:08.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_23 2026-02-20T21:39:08.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:39:08.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-02-20T21:39:08.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-02-20T21:39:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:08.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_23 deleted '' 2026-02-20T21:39:08.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:08.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:08.988 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_23 2026-02-20T21:39:08.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:08.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:08.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:08.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:08.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_23 2026-02-20T21:39:08.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:08.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:08.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_23 2026-02-20T21:39:08.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_23 2026-02-20T21:39:08.989 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.009 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_23: (2) No such file or directory 2026-02-20T21:39:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_24 2026-02-20T21:39:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_24 deleted 2026-02-20T21:39:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:39:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_24 2026-02-20T21:39:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.013 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-20T21:39:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-02-20T21:39:09.014 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-02-20T21:39:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:09.037 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_24 deleted '' 2026-02-20T21:39:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_24 2026-02-20T21:39:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_24 2026-02-20T21:39:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_24 2026-02-20T21:39:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_24 2026-02-20T21:39:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_24: (2) No such file or directory 2026-02-20T21:39:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_25 2026-02-20T21:39:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_25 deleted 2026-02-20T21:39:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:39:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_25 2026-02-20T21:39:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:39:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-02-20T21:39:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-02-20T21:39:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:09.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_25 deleted '' 2026-02-20T21:39:09.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_25 2026-02-20T21:39:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_25 2026-02-20T21:39:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_25 2026-02-20T21:39:09.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_25 2026-02-20T21:39:09.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_25: (2) No such file or directory 2026-02-20T21:39:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_26 2026-02-20T21:39:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_26 deleted 2026-02-20T21:39:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:39:09.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_26 2026-02-20T21:39:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:39:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-02-20T21:39:09.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-02-20T21:39:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:09.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:09.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_26 deleted '' 2026-02-20T21:39:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_26 2026-02-20T21:39:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_26 2026-02-20T21:39:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_26 2026-02-20T21:39:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_26 2026-02-20T21:39:09.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_26: (2) No such file or directory 2026-02-20T21:39:09.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:09.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:09.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:09.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:09.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_27 2026-02-20T21:39:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_27 deleted 2026-02-20T21:39:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:39:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_27 2026-02-20T21:39:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:39:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-02-20T21:39:09.466 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-02-20T21:39:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:09.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_27 deleted '' 2026-02-20T21:39:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_27 2026-02-20T21:39:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:09.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_27 2026-02-20T21:39:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_27 2026-02-20T21:39:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_27 2026-02-20T21:39:09.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_27: (2) No such file or directory 2026-02-20T21:39:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_28 2026-02-20T21:39:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_28 deleted 2026-02-20T21:39:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:39:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_28 2026-02-20T21:39:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:39:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-02-20T21:39:09.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.637 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-02-20T21:39:09.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:09.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:09.639 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:09.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_28 deleted '' 2026-02-20T21:39:09.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_28 2026-02-20T21:39:09.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:09.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:09.741 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:09.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_28 2026-02-20T21:39:09.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_28 2026-02-20T21:39:09.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_28 2026-02-20T21:39:09.742 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_28: (2) No such file or directory 2026-02-20T21:39:09.768 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_29 2026-02-20T21:39:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_29 deleted 2026-02-20T21:39:09.769 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:39:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_29 2026-02-20T21:39:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:39:09.770 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-02-20T21:39:09.771 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-02-20T21:39:09.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:09.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:09.791 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:09.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_29 deleted '' 2026-02-20T21:39:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_29 2026-02-20T21:39:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_29 2026-02-20T21:39:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.894 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_29 2026-02-20T21:39:09.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_29 2026-02-20T21:39:09.895 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_29: (2) No such file or directory 2026-02-20T21:39:09.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:09.917 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_30 2026-02-20T21:39:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_30 deleted 2026-02-20T21:39:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:39:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:09.918 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:09.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:09.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_30 2026-02-20T21:39:09.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:09.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:09.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:39:09.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-02-20T21:39:09.919 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:09.938 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-02-20T21:39:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:09.940 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_30 deleted '' 2026-02-20T21:39:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_30 2026-02-20T21:39:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_30 2026-02-20T21:39:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_30 2026-02-20T21:39:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_30 2026-02-20T21:39:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_30: (2) No such file or directory 2026-02-20T21:39:10.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:10.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_31 2026-02-20T21:39:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_31 deleted 2026-02-20T21:39:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:39:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:10.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_31 2026-02-20T21:39:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:39:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-02-20T21:39:10.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-02-20T21:39:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_31 deleted '' 2026-02-20T21:39:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_31 2026-02-20T21:39:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:10.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_31 2026-02-20T21:39:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_31 2026-02-20T21:39:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_31 2026-02-20T21:39:10.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.214 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_31: (2) No such file or directory 2026-02-20T21:39:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_32 2026-02-20T21:39:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_32 deleted 2026-02-20T21:39:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:39:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.218 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_32 2026-02-20T21:39:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:39:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-02-20T21:39:10.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.240 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-02-20T21:39:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:10.242 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_32 deleted '' 2026-02-20T21:39:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_32 2026-02-20T21:39:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_32 2026-02-20T21:39:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_32 2026-02-20T21:39:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_32 2026-02-20T21:39:10.344 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_32: (2) No such file or directory 2026-02-20T21:39:10.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:10.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:10.368 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_33 2026-02-20T21:39:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_33 deleted 2026-02-20T21:39:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:39:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_33 2026-02-20T21:39:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:39:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-02-20T21:39:10.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.388 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-02-20T21:39:10.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:10.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:10.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_33 deleted '' 2026-02-20T21:39:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_33 2026-02-20T21:39:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_33 2026-02-20T21:39:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_33 2026-02-20T21:39:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_33 2026-02-20T21:39:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_33: (2) No such file or directory 2026-02-20T21:39:10.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:10.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:10.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:10.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:10.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_34 2026-02-20T21:39:10.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_34 deleted 2026-02-20T21:39:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:39:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_34 2026-02-20T21:39:10.520 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:39:10.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-02-20T21:39:10.521 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-02-20T21:39:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:10.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:10.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:10.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_34 deleted '' 2026-02-20T21:39:10.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_34 2026-02-20T21:39:10.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:10.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:10.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:10.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_34 2026-02-20T21:39:10.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_34 2026-02-20T21:39:10.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_34 2026-02-20T21:39:10.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_34: (2) No such file or directory 2026-02-20T21:39:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:10.671 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_35 2026-02-20T21:39:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_35 deleted 2026-02-20T21:39:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:39:10.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_35 2026-02-20T21:39:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:39:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-02-20T21:39:10.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-02-20T21:39:10.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:10.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:10.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:10.796 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_35 deleted '' 2026-02-20T21:39:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_35 2026-02-20T21:39:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_35 2026-02-20T21:39:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_35 2026-02-20T21:39:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_35 2026-02-20T21:39:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_35: (2) No such file or directory 2026-02-20T21:39:10.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:10.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:10.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:10.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:10.822 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_36 2026-02-20T21:39:10.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_36 deleted 2026-02-20T21:39:10.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:39:10.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:10.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:10.823 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_36 2026-02-20T21:39:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:39:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-02-20T21:39:10.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-02-20T21:39:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:10.947 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_36 deleted '' 2026-02-20T21:39:10.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_36 2026-02-20T21:39:10.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:10.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:10.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:10.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_36 2026-02-20T21:39:10.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.948 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_36 2026-02-20T21:39:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_36 2026-02-20T21:39:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_36: (2) No such file or directory 2026-02-20T21:39:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:10.972 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_37 2026-02-20T21:39:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_37 deleted 2026-02-20T21:39:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:39:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:10.973 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_37 2026-02-20T21:39:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:39:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-02-20T21:39:10.974 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:10.996 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-02-20T21:39:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:10.998 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_37 deleted '' 2026-02-20T21:39:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_37 2026-02-20T21:39:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_37 2026-02-20T21:39:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_37 2026-02-20T21:39:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_37 2026-02-20T21:39:11.100 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_37: (2) No such file or directory 2026-02-20T21:39:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:11.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_38 2026-02-20T21:39:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_38 deleted 2026-02-20T21:39:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:39:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_38 2026-02-20T21:39:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:11.127 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:11.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:39:11.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-02-20T21:39:11.128 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:11.149 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-02-20T21:39:11.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:11.151 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:11.152 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_38 deleted '' 2026-02-20T21:39:11.253 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_38 2026-02-20T21:39:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_38 2026-02-20T21:39:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_38 2026-02-20T21:39:11.254 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_38 2026-02-20T21:39:11.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:11.275 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_38: (2) No such file or directory 2026-02-20T21:39:11.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:11.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:11.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:11.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:11.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_39 2026-02-20T21:39:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_39 deleted 2026-02-20T21:39:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:39:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_39 2026-02-20T21:39:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:11.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:11.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:39:11.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-02-20T21:39:11.280 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:11.299 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-02-20T21:39:11.300 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:11.301 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:11.402 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_39 deleted '' 2026-02-20T21:39:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_39 2026-02-20T21:39:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:11.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:11.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_39 2026-02-20T21:39:11.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:11.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:11.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_39 2026-02-20T21:39:11.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_39 2026-02-20T21:39:11.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:11.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_39: (2) No such file or directory 2026-02-20T21:39:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_40 2026-02-20T21:39:11.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_40 deleted 2026-02-20T21:39:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:39:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_40 2026-02-20T21:39:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:11.633 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:11.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:39:11.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-02-20T21:39:11.634 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:11.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-02-20T21:39:11.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:11.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:11.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_40 deleted '' 2026-02-20T21:39:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_40 2026-02-20T21:39:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_40 2026-02-20T21:39:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_40 2026-02-20T21:39:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_40 2026-02-20T21:39:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:11.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_40: (2) No such file or directory 2026-02-20T21:39:11.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:11.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:11.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:11.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:11.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:11.783 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_41 2026-02-20T21:39:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_41 deleted 2026-02-20T21:39:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:39:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_41 2026-02-20T21:39:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:11.784 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:11.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:39:11.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-02-20T21:39:11.785 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-02-20T21:39:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:11.807 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:11.908 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_41 deleted '' 2026-02-20T21:39:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_41 2026-02-20T21:39:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_41 2026-02-20T21:39:11.909 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_41 2026-02-20T21:39:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_41 2026-02-20T21:39:11.910 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:11.931 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_41: (2) No such file or directory 2026-02-20T21:39:11.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:11.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:11.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:11.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:11.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:11.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_42 2026-02-20T21:39:11.934 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_42 deleted 2026-02-20T21:39:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:39:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_42 2026-02-20T21:39:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:11.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-20T21:39:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-02-20T21:39:11.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:12.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-02-20T21:39:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_42 deleted '' 2026-02-20T21:39:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_42 2026-02-20T21:39:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:12.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_42 2026-02-20T21:39:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_42 2026-02-20T21:39:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_42 2026-02-20T21:39:12.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:12.285 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_42: (2) No such file or directory 2026-02-20T21:39:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_43 2026-02-20T21:39:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_43 deleted 2026-02-20T21:39:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:12.289 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:39:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_43 2026-02-20T21:39:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:12.290 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:39:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-02-20T21:39:12.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:12.311 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-02-20T21:39:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:12.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_43 deleted '' 2026-02-20T21:39:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:12.415 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_43 2026-02-20T21:39:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_43 2026-02-20T21:39:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_43 2026-02-20T21:39:12.416 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_43 2026-02-20T21:39:12.417 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:12.437 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_43: (2) No such file or directory 2026-02-20T21:39:12.440 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_44 2026-02-20T21:39:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_44 deleted 2026-02-20T21:39:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:12.441 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:39:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_44 2026-02-20T21:39:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:39:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-02-20T21:39:12.442 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:12.462 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-02-20T21:39:12.463 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:12.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:12.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:12.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_44 deleted '' 2026-02-20T21:39:12.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_44 2026-02-20T21:39:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_44 2026-02-20T21:39:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_44 2026-02-20T21:39:12.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_44 2026-02-20T21:39:12.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:12.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_44: (2) No such file or directory 2026-02-20T21:39:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_45 2026-02-20T21:39:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_45 deleted 2026-02-20T21:39:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:12.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:39:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_45 2026-02-20T21:39:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:39:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-02-20T21:39:12.591 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:12.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-02-20T21:39:12.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:12.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:12.612 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_45 deleted '' 2026-02-20T21:39:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_45 2026-02-20T21:39:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:12.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:12.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_45 2026-02-20T21:39:12.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:12.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:12.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_45 2026-02-20T21:39:12.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_45 2026-02-20T21:39:12.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:12.735 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_45: (2) No such file or directory 2026-02-20T21:39:12.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:12.738 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:12.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:12.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:12.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:12.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_46 2026-02-20T21:39:12.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_46 deleted 2026-02-20T21:39:12.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:12.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:12.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:39:12.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:12.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:12.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:12.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_46 2026-02-20T21:39:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:39:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-02-20T21:39:12.740 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:12.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-02-20T21:39:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:12.763 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:12.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_46 deleted '' 2026-02-20T21:39:12.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:12.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:12.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_46 2026-02-20T21:39:12.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:12.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:12.865 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:12.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:12.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_46 2026-02-20T21:39:12.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:12.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:12.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_46 2026-02-20T21:39:12.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_46 2026-02-20T21:39:12.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:12.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_46: (2) No such file or directory 2026-02-20T21:39:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_47 2026-02-20T21:39:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_47 deleted 2026-02-20T21:39:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:12.892 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_47 2026-02-20T21:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-20T21:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-02-20T21:39:12.893 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:12.912 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-02-20T21:39:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_47 deleted '' 2026-02-20T21:39:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_47 2026-02-20T21:39:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:13.016 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_47 2026-02-20T21:39:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_47 2026-02-20T21:39:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_47 2026-02-20T21:39:13.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:13.039 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_47: (2) No such file or directory 2026-02-20T21:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_48 2026-02-20T21:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_48 deleted 2026-02-20T21:39:13.042 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:39:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_48 2026-02-20T21:39:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:39:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-02-20T21:39:13.043 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-02-20T21:39:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:13.066 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:13.067 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:13.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_48 deleted '' 2026-02-20T21:39:13.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:13.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:13.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_48 2026-02-20T21:39:13.168 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_48 2026-02-20T21:39:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_48 2026-02-20T21:39:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_48 2026-02-20T21:39:13.169 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_48: (2) No such file or directory 2026-02-20T21:39:13.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:13.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:13.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:13.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:13.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:13.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_49 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_49 deleted 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_49 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:39:13.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-02-20T21:39:13.195 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:13.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-02-20T21:39:13.215 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:13.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:13.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:13.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_49 deleted '' 2026-02-20T21:39:13.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:13.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:13.317 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_49 2026-02-20T21:39:13.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:13.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:13.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:13.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:13.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_49 2026-02-20T21:39:13.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:13.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:13.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_49 2026-02-20T21:39:13.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_49 2026-02-20T21:39:13.318 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:13.339 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_49: (2) No such file or directory 2026-02-20T21:39:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 ${IMAGE_COUNT}` 2026-02-20T21:39:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_50 2026-02-20T21:39:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_50 deleted 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_50 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-02-20T21:39:13.343 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:13.364 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-02-20T21:39:13.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:13.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:13.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:13.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_50 deleted '' 2026-02-20T21:39:13.467 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_50 2026-02-20T21:39:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_50 2026-02-20T21:39:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_50 2026-02-20T21:39:13.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_50 2026-02-20T21:39:13.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:13.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_50: (2) No such file or directory 2026-02-20T21:39:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: delete images during bootstrap' 2026-02-20T21:39:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.IQNOxgiLiE/rbd-mirror.test.log 2026-02-20T21:39:13.495 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-20T21:39:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-20 21:39:13 TEST: delete images during bootstrap 2026-02-20T21:39:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-20 21:39:13 TEST: delete images during bootstrap 2026-02-20T21:39:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_pool_mirror_mode cluster1 mirror image 2026-02-20T21:39:13.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:13.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:13.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=image 2026-02-20T21:39:13.497 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster1 mirror pool enable mirror image 2026-02-20T21:39:13.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:note: changing mirroring mode from pool to image 2026-02-20T21:39:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-02-20T21:39:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=image 2026-02-20T21:39:13.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-02-20T21:39:13.545 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:note: changing mirroring mode from pool to image 2026-02-20T21:39:13.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ start_mirror cluster1 2026-02-20T21:39:13.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:13.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance 2026-02-20T21:39:13.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set_cluster_instance cluster1 cluster instance 2026-02-20T21:39:13.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local val=cluster1 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster_var_name=cluster 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local instance_var_name=instance 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cluster=cluster1 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=cluster1 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' cluster1 = cluster1 ']' 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ instance=0 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval cluster=cluster1 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ cluster=cluster1 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ eval instance=0 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ instance=0 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:39:13.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=test 2026-02-20T21:39:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 1 10 2026-02-20T21:39:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:39:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_1 2026-02-20T21:39:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_1 512M 2026-02-20T21:39:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:39:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:39:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:39:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:39:13.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-20T21:39:13.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-20T21:39:13.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:39:13.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_1 2026-02-20T21:39:13.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_1 2026-02-20T21:39:13.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:13.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:13.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:13.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:39:13.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_1 journal 2026-02-20T21:39:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-20T21:39:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_1 2026-02-20T21:39:13.643 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_1': 2026-02-20T21:39:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:39:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:39:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-20T21:39:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 304e32af5c3f 2026-02-20T21:39:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.304e32af5c3f 2026-02-20T21:39:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:39:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:39:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:39:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:39:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:39:13 2026 2026-02-20T21:39:13.644 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:39:13 2026 2026-02-20T21:39:13.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:39:13 2026 2026-02-20T21:39:13.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 304e32af5c3f 2026-02-20T21:39:13.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:39:13.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:39:13.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: b23a18fc-a6b0-4892-bd9f-46ab64c6e761 2026-02-20T21:39:13.645 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-20T21:39:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_1 2026-02-20T21:39:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:13.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:39:13.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=8 2026-02-20T21:39:13.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:39:13.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 8s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_1 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:39:14.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 29748224 bytes 2026-02-20T21:39:14.694 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 42835968 bytes 2026-02-20T21:39:15.017 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 60477440 bytes 2026-02-20T21:39:15.197 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:39:15.196+0000 7f03c8435640 -1 reset not still connected to 0x55af9ee28d00 2026-02-20T21:39:15.198 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:39:15.196+0000 7f03c8435640 -1 reset not still connected to 0x55afbc1c64e0 2026-02-20T21:39:15.208 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 89832448 bytes 2026-02-20T21:39:15.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 151643136 bytes 2026-02-20T21:39:17.053 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 211340288 bytes 2026-02-20T21:39:17.664 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 253346816 bytes 2026-02-20T21:39:17.889 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 268831744 bytes 2026-02-20T21:39:17.945 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 274761728 bytes 2026-02-20T21:39:19.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 391242752 bytes 2026-02-20T21:39:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 446170112 bytes 2026-02-20T21:39:21.355 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 501524480 bytes 2026-02-20T21:39:21.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:39:21.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:39:21.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:39:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_1 present 2026-02-20T21:39:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-20T21:39:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:21.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:21.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:21.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_1 2026-02-20T21:39:21.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:21.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:21.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:39:21.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-20T21:39:21.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=1212138ee1a4 2026-02-20T21:39:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_1 present 1212138ee1a4 2026-02-20T21:39:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:39:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=1212138ee1a4 2026-02-20T21:39:21.789 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_1 2026-02-20T21:39:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:39:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-20T21:39:21.790 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:21.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=1212138ee1a4 2026-02-20T21:39:21.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1212138ee1a4 2026-02-20T21:39:21.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 1212138ee1a4 -o 1212138ee1a4 = 1212138ee1a4 2026-02-20T21:39:21.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-20T21:39:21.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-20T21:39:21.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:21.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_1 2026-02-20T21:39:21.818 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:21.819 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_1 2026-02-20T21:39:27.003 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:39:27.001+0000 7faa4d82c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x561ec01b8900 finish: resending after timeout 2026-02-20T21:39:32.003 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:39:32.001+0000 7faa4d02b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x561ec01b8900 finish: resending after timeout 2026-02-20T21:39:35.390 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.919151950559507) > weight (0.25) 2026-02-20T21:39:35.391 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.9984395214567671) > weight (0.25) 2026-02-20T21:39:35.391 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.2 2026-02-20T21:39:35.391 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Sent signal 15 2026-02-20T21:39:35.391 INFO:tasks.rbd_mirror_thrash:waiting for 2.2914234399169797 secs before reviving daemons 2026-02-20T21:39:35.392 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:39:35.390+0000 7fb0103aa640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 94122) UID: 1000 2026-02-20T21:39:37.004 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:39:37.002+0000 7faa4d82c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x561ec01b8900 finish: resending after timeout 2026-02-20T21:39:37.684 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.2 2026-02-20T21:39:37.684 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:39:37.684 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.2 2026-02-20T21:39:37.685 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Restarting daemon 2026-02-20T21:39:37.685 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 2026-02-20T21:39:37.687 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Started 2026-02-20T21:39:37.687 INFO:tasks.rbd_mirror_thrash:waiting for 113.72483119402085 secs before thrashing 2026-02-20T21:39:42.004 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:39:42.003+0000 7faa4d02b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x561ec01b8900 finish: resending after timeout 2026-02-20T21:39:47.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:39:47.003+0000 7faa4d82c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x561ec01b8900 finish: resending after timeout 2026-02-20T21:39:48.772 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:39:48.769+0000 7f8f77c3b640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x562e90732b60 handle_prepare_local_image: error preparing local image for replay: (85) Interrupted system call should be restarted 2026-02-20T21:39:48.772 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:39:48.769+0000 7f8f77c3b640 -1 rbd::mirror::ImageReplayer: 0x562e9036b180 [3/b23a18fc-a6b0-4892-bd9f-46ab64c6e761] operator(): start failed: (85) Interrupted system call should be restarted 2026-02-20T21:39:52.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:39:52.002+0000 7faa4d02b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x561ec01b8900 finish: resending after timeout 2026-02-20T21:39:54.069 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:39:54.067+0000 7faa40011640 -1 rbd::mirror::image_replayer::journal::SyncPointHandler: 0x561ebffe7cb0 handle_update_sync_points: failed to update remote journal client meta for image b23a18fc-a6b0-4892-bd9f-46ab64c6e761: (2) No such file or directory 2026-02-20T21:39:54.070 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:39:54.067+0000 7faa40011640 -1 rbd::mirror::image_sync::SyncPointPruneRequest: 0x561ec4111c20 handle_update_sync_points: failed to update client data: (2) No such file or directory 2026-02-20T21:39:54.070 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:39:54.067+0000 7faa40011640 -1 rbd::mirror::ImageSync: 0x561ec31c1ce0 handle_prune_sync_points: failed to prune sync point: (2) No such file or directory 2026-02-20T21:39:54.070 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:39:54.067+0000 7faa40011640 -1 rbd::mirror::image_replayer::BootstrapRequest: 0x561ee01ddd40 handle_image_sync: failed to sync remote image: (2) No such file or directory 2026-02-20T21:39:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-20T21:39:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_1 deleted 2026-02-20T21:39:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:39:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:39:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:54.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_1 2026-02-20T21:39:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:39:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-20T21:39:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:54.157 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-20T21:39:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:39:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:39:54.159 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:39:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_1 deleted '' 2026-02-20T21:39:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:39:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:39:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:39:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:39:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:39:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_1 2026-02-20T21:39:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:39:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:39:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_1 2026-02-20T21:39:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_1 2026-02-20T21:39:54.262 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:39:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_1: (2) No such file or directory 2026-02-20T21:39:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:39:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:39:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:39:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_1 2026-02-20T21:39:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_1 2026-02-20T21:39:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_1 2026-02-20T21:39:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:54.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:39:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:39:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_1 2026-02-20T21:39:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_1 2026-02-20T21:39:54.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_1 2026-02-20T21:39:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_1 2026-02-20T21:39:54.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:39:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:39:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:39:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_2 2026-02-20T21:39:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_2 512M 2026-02-20T21:39:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:39:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:39:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:39:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:39:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:39:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-20T21:39:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-20T21:39:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:39:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_2 2026-02-20T21:39:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_2 2026-02-20T21:39:54.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:54.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:54.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:39:54.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:39:54.492 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_2 journal 2026-02-20T21:39:54.524 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-20T21:39:54.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_2 2026-02-20T21:39:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_2': 2026-02-20T21:39:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:39:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:39:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-20T21:39:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 306c3b405ad0 2026-02-20T21:39:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.306c3b405ad0 2026-02-20T21:39:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:39:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:39:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:39:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:39:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:39:54 2026 2026-02-20T21:39:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:39:54 2026 2026-02-20T21:39:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:39:54 2026 2026-02-20T21:39:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 306c3b405ad0 2026-02-20T21:39:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:39:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:39:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 01515c23-8cee-4030-82c4-b68023f76d44 2026-02-20T21:39:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-20T21:39:54.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_2 2026-02-20T21:39:54.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:39:54.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:39:54.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:39:54.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:39:54.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=9 2026-02-20T21:39:54.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:39:54.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 9s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_2 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:39:55.370 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 40572928 bytes 2026-02-20T21:39:56.611 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 134301696 bytes 2026-02-20T21:39:57.786 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 212564992 bytes 2026-02-20T21:39:58.024 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 227709952 bytes 2026-02-20T21:39:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 293928960 bytes 2026-02-20T21:39:59.739 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 344757248 bytes 2026-02-20T21:40:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 361464832 bytes 2026-02-20T21:40:00.860 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 385806336 bytes 2026-02-20T21:40:01.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 444059648 bytes 2026-02-20T21:40:03.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 514147328 bytes 2026-02-20T21:40:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:40:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:40:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:40:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:40:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_2 present 2026-02-20T21:40:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:40:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:03.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-20T21:40:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:40:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:40:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:40:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T21:40:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:40:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:40:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:40:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T21:40:03.564 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:40:03.588 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T21:40:03.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:40:03.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:40:03.590 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:40:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_2 present '' 2026-02-20T21:40:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:40:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:03.692 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:40:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:40:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:40:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:40:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T21:40:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:40:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:40:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:40:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T21:40:03.693 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:40:03.715 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T21:40:03.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:40:03.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:40:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = deleted 2026-02-20T21:40:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:40:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:40:04.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_2 present '' 2026-02-20T21:40:04.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:40:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:40:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:40:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:40:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:40:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T21:40:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:40:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:40:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:40:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T21:40:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:40:04.745 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T21:40:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:40:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:40:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = deleted 2026-02-20T21:40:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:40:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 2 2026-02-20T21:40:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_2 present '' 2026-02-20T21:40:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:40:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:06.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:06.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:40:06.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:40:06.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:40:06.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:40:06.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T21:40:06.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:40:06.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:40:06.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:40:06.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T21:40:06.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:40:06.776 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T21:40:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:40:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:40:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = deleted 2026-02-20T21:40:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:40:06.778 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 4 2026-02-20T21:40:10.779 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_2 present '' 2026-02-20T21:40:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:40:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:40:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:40:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:40:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:40:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T21:40:10.780 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:40:10.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:40:10.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:40:10.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T21:40:10.781 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:40:10.806 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T21:40:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:40:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:40:10.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = deleted 2026-02-20T21:40:10.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:40:10.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 8 2026-02-20T21:40:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_2 present '' 2026-02-20T21:40:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:40:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:40:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:40:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:40:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:40:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T21:40:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:40:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:40:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:40:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T21:40:18.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:40:18.837 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T21:40:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:40:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:40:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = deleted 2026-02-20T21:40:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:40:18.839 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 8 2026-02-20T21:40:25.200 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:40:25.198+0000 7faa4d02b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x561ebea89bc0 finish: resending after timeout 2026-02-20T21:40:26.840 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_2 present '' 2026-02-20T21:40:26.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:40:26.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:26.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:26.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:40:26.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:40:26.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:40:26.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:40:26.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T21:40:26.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:40:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:40:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:40:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T21:40:26.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:40:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=1212a4ac5f00 2026-02-20T21:40:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1212a4ac5f00 2026-02-20T21:40:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z '' -o '' = 1212a4ac5f00 2026-02-20T21:40:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-20T21:40:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-20T21:40:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:40:26.873 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_2 2026-02-20T21:40:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:40:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:26.874 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_2 2026-02-20T21:40:30.200 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:40:30.198+0000 7faa4d82c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x561ebea89bc0 finish: resending after timeout 2026-02-20T21:40:35.200 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:40:35.199+0000 7faa4d02b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x561ebea89bc0 finish: resending after timeout 2026-02-20T21:40:40.201 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:40:40.198+0000 7faa4d82c640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x561ebea89bc0 finish: resending after timeout 2026-02-20T21:40:45.201 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:40:45.199+0000 7faa4d02b640 -1 rbd::mirror::InstanceWatcher: C_NotifyInstanceRequest: 0x561ebea89bc0 finish: resending after timeout 2026-02-20T21:40:47.572 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.8611109907031348) > weight (0.25) 2026-02-20T21:40:47.572 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.8661314024540355) > weight (0.25) 2026-02-20T21:40:47.572 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.2 with skip (0.778234975790411) > weight (0.25) 2026-02-20T21:40:47.572 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.3 with skip (0.8204120651258173) > weight (0.25) 2026-02-20T21:40:47.572 INFO:tasks.rbd_mirror_thrash:waiting for 116.06599057890384 secs before thrashing 2026-02-20T21:40:49.307 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-20T21:40:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_2 deleted 2026-02-20T21:40:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:40:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:40:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:40:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:40:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:40:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T21:40:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:40:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:40:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:40:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T21:40:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:40:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=1212a4ac5f00 2026-02-20T21:40:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:40:49.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:40:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_2 deleted 1212a4ac5f00 2026-02-20T21:40:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:40:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:40:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=1212a4ac5f00 2026-02-20T21:40:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:40:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:40:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T21:40:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:40:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:40:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:40:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T21:40:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:40:49.474 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=1212a4ac5f00 2026-02-20T21:40:49.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1212a4ac5f00 2026-02-20T21:40:49.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 1212a4ac5f00 -o 1212a4ac5f00 = 1212a4ac5f00 2026-02-20T21:40:49.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-20T21:40:49.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = present 2026-02-20T21:40:49.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:40:49.475 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 1 2026-02-20T21:40:50.476 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_2 deleted 1212a4ac5f00 2026-02-20T21:40:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:40:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:40:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=1212a4ac5f00 2026-02-20T21:40:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:40:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:40:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_2 2026-02-20T21:40:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:40:50.477 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:40:50.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_2 2026-02-20T21:40:50.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_2 2026-02-20T21:40:50.478 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:40:50.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_2: (2) No such file or directory 2026-02-20T21:40:50.501 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:40:50.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:40:50.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:40:50.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:40:50.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_2 2026-02-20T21:40:50.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:40:50.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:50.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:50.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_2 2026-02-20T21:40:51.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-20T21:40:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_2 2026-02-20T21:40:51.073 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:40:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:40:51.074 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:40:51.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_2 2026-02-20T21:40:51.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:40:51.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:51.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_2 2026-02-20T21:40:51.075 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_2 2026-02-20T21:40:51.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_2 2026-02-20T21:40:51.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:40:51.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:40:51.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:40:51.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_3 2026-02-20T21:40:51.229 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_3 512M 2026-02-20T21:40:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:40:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:40:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:40:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:40:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:40:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:40:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-20T21:40:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-20T21:40:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:40:51.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_3 2026-02-20T21:40:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_3 2026-02-20T21:40:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:40:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:40:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:40:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_3 journal 2026-02-20T21:40:51.295 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-20T21:40:51.298 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_3 2026-02-20T21:40:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_3': 2026-02-20T21:40:51.324 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:40:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:40:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-20T21:40:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 3084b3b30232 2026-02-20T21:40:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.3084b3b30232 2026-02-20T21:40:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:40:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:40:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:40:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:40:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:40:51 2026 2026-02-20T21:40:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:40:51 2026 2026-02-20T21:40:51.325 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:40:51 2026 2026-02-20T21:40:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 3084b3b30232 2026-02-20T21:40:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:40:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:40:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 3e97d636-4a7c-4b88-a6df-b237d860145c 2026-02-20T21:40:51.326 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-20T21:40:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_3 2026-02-20T21:40:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:40:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:40:51.327 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:40:51.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=5 2026-02-20T21:40:51.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:40:51.328 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 5s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_3 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:40:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 88281088 bytes 2026-02-20T21:40:54.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 170687488 bytes 2026-02-20T21:40:55.486 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 211544064 bytes 2026-02-20T21:40:56.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 254823424 bytes 2026-02-20T21:40:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:40:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:40:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:40:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:40:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_3 present 2026-02-20T21:40:56.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:40:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:40:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-20T21:40:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:40:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:40:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:40:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_3 2026-02-20T21:40:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:40:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:40:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:40:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-20T21:40:56.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:40:56.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=12128cd2c6c1 2026-02-20T21:40:56.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:40:56.367 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:40:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_3 present 12128cd2c6c1 2026-02-20T21:40:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:40:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:40:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:40:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=12128cd2c6c1 2026-02-20T21:40:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:40:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:40:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_3 2026-02-20T21:40:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:40:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:40:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:40:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-20T21:40:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:40:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=12128cd2c6c1 2026-02-20T21:40:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 12128cd2c6c1 2026-02-20T21:40:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 12128cd2c6c1 -o 12128cd2c6c1 = 12128cd2c6c1 2026-02-20T21:40:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-20T21:40:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-20T21:40:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:40:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_3 2026-02-20T21:40:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:40:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:40:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:40:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_3 2026-02-20T21:41:25.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-20T21:41:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_3 deleted 2026-02-20T21:41:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:41:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:41:25.105 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:41:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:41:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:41:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:41:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:41:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_3 2026-02-20T21:41:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:41:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:41:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:41:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-20T21:41:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:41:25.129 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-20T21:41:25.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:41:25.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:41:25.131 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:41:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_3 deleted '' 2026-02-20T21:41:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:41:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:41:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:41:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:41:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:41:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:41:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:41:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_3 2026-02-20T21:41:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:41:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:41:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_3 2026-02-20T21:41:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_3 2026-02-20T21:41:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:41:25.255 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_3: (2) No such file or directory 2026-02-20T21:41:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:41:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:41:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:41:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:41:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_3 2026-02-20T21:41:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:41:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:41:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:41:25.258 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_3 2026-02-20T21:41:25.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_3 2026-02-20T21:41:25.286 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:41:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:41:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:41:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:41:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:41:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_3 2026-02-20T21:41:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:41:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:41:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_3 2026-02-20T21:41:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_3 2026-02-20T21:41:25.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_3 2026-02-20T21:41:25.425 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:41:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:41:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:41:25.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_4 2026-02-20T21:41:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_4 512M 2026-02-20T21:41:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:41:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:41:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:41:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:41:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:41:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:41:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:41:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-20T21:41:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-20T21:41:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:41:25.430 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_4 2026-02-20T21:41:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_4 2026-02-20T21:41:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:41:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:41:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:41:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:41:25.468 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_4 journal 2026-02-20T21:41:25.499 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-20T21:41:25.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_4 2026-02-20T21:41:25.527 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_4': 2026-02-20T21:41:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:41:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:41:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-20T21:41:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 309c3c769b6f 2026-02-20T21:41:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.309c3c769b6f 2026-02-20T21:41:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:41:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:41:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:41:25.528 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:41:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:41:25 2026 2026-02-20T21:41:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:41:25 2026 2026-02-20T21:41:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:41:25 2026 2026-02-20T21:41:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 309c3c769b6f 2026-02-20T21:41:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:41:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:41:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 87204363-8aad-4461-ae3b-8f88113d6ea5 2026-02-20T21:41:25.529 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-20T21:41:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_4 2026-02-20T21:41:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:41:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:41:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:41:25.530 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:41:25.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-20T21:41:25.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:41:25.532 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_4 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:41:26.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 46867456 bytes 2026-02-20T21:41:26.762 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 65711104 bytes 2026-02-20T21:41:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 113162240 bytes 2026-02-20T21:41:28.088 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 143857664 bytes 2026-02-20T21:41:28.714 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 177575936 bytes 2026-02-20T21:41:29.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 225699840 bytes 2026-02-20T21:41:30.233 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 282514432 bytes 2026-02-20T21:41:31.022 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 312712192 bytes 2026-02-20T21:41:31.413 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5528217580589985) > weight (0.25) 2026-02-20T21:41:31.413 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.51058093104153) > weight (0.25) 2026-02-20T21:41:31.413 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.9827964356682849) > weight (0.25) 2026-02-20T21:41:31.413 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.3 with skip (0.6181795251784179) > weight (0.25) 2026-02-20T21:41:31.413 INFO:tasks.rbd_mirror_thrash:waiting for 66.23855921494571 secs before thrashing 2026-02-20T21:41:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 412463104 bytes 2026-02-20T21:41:32.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 471384064 bytes 2026-02-20T21:41:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 520827904 bytes 2026-02-20T21:41:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 572187648 bytes 2026-02-20T21:41:35.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 620326912 bytes 2026-02-20T21:41:36.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 697086976 bytes 2026-02-20T21:41:37.132 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 733110272 bytes 2026-02-20T21:41:38.069 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 787769344 bytes 2026-02-20T21:41:38.348 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 805244928 bytes 2026-02-20T21:41:39.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 861734912 bytes 2026-02-20T21:41:39.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:41:39.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:41:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:41:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:41:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_4 present 2026-02-20T21:41:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:41:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:41:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:41:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-20T21:41:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:41:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:41:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:41:39.543 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_4 2026-02-20T21:41:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:41:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:41:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:41:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-20T21:41:39.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:41:39.576 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=12126f4744c6 2026-02-20T21:41:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:41:39.577 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:41:39.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_4 present 12126f4744c6 2026-02-20T21:41:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:41:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:41:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:41:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:41:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=12126f4744c6 2026-02-20T21:41:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:41:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:41:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_4 2026-02-20T21:41:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:41:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:41:39.679 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:41:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-20T21:41:39.680 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:41:39.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=12126f4744c6 2026-02-20T21:41:39.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 12126f4744c6 2026-02-20T21:41:39.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 12126f4744c6 -o 12126f4744c6 = 12126f4744c6 2026-02-20T21:41:39.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-20T21:41:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-20T21:41:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:41:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_4 2026-02-20T21:41:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:41:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:41:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:41:39.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_4 2026-02-20T21:42:10.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-20T21:42:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_4 deleted 2026-02-20T21:42:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:42:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:42:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:42:10.243 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:42:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:42:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:42:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_4 2026-02-20T21:42:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:42:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:42:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:42:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-20T21:42:10.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:42:10.268 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-20T21:42:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:42:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:42:10.270 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:42:10.372 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_4 deleted '' 2026-02-20T21:42:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:42:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:42:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:42:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:42:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:42:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:42:10.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_4 2026-02-20T21:42:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:42:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:42:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_4 2026-02-20T21:42:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_4 2026-02-20T21:42:10.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:42:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_4: (2) No such file or directory 2026-02-20T21:42:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:42:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:42:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:42:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:42:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_4 2026-02-20T21:42:10.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:10.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:10.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:42:10.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_4 2026-02-20T21:42:10.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_4 2026-02-20T21:42:10.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:10.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:10.426 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:42:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:42:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:42:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_4 2026-02-20T21:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_4 2026-02-20T21:42:10.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_4 2026-02-20T21:42:10.453 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_4 2026-02-20T21:42:10.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:42:10.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:42:10.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:42:10.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_5 2026-02-20T21:42:10.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_5 512M 2026-02-20T21:42:10.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:10.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:42:10.565 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:10.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:42:10.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:42:10.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:42:10.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:42:10.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-20T21:42:10.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-20T21:42:10.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:42:10.566 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_5 2026-02-20T21:42:10.601 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_5 2026-02-20T21:42:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:42:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:42:10.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_5 journal 2026-02-20T21:42:10.631 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-20T21:42:10.635 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_5 2026-02-20T21:42:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_5': 2026-02-20T21:42:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:42:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:42:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-20T21:42:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 30b423d478e6 2026-02-20T21:42:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.30b423d478e6 2026-02-20T21:42:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:42:10.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:42:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:42:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:42:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:42:10 2026 2026-02-20T21:42:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:42:10 2026 2026-02-20T21:42:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:42:10 2026 2026-02-20T21:42:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 30b423d478e6 2026-02-20T21:42:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:42:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:42:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 92d8ec42-d5ab-4da5-a311-bf3d32aaf71e 2026-02-20T21:42:10.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-20T21:42:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_5 2026-02-20T21:42:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:42:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:42:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=13 2026-02-20T21:42:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:42:10.663 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 13s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_5 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:42:11.359 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 26742784 bytes 2026-02-20T21:42:11.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 31846400 bytes 2026-02-20T21:42:12.071 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 71889920 bytes 2026-02-20T21:42:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 119229440 bytes 2026-02-20T21:42:14.429 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 168753152 bytes 2026-02-20T21:42:14.935 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 219172864 bytes 2026-02-20T21:42:15.318 INFO:tasks.ceph.cluster2.osd.0.trial163.stderr:2026-02-20T21:42:15.316+0000 7f03c8435640 -1 reset not still connected to 0x55af9837add0 2026-02-20T21:42:15.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 240556032 bytes 2026-02-20T21:42:15.775 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 279250944 bytes 2026-02-20T21:42:16.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 301670400 bytes 2026-02-20T21:42:16.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 312097792 bytes 2026-02-20T21:42:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 365945856 bytes 2026-02-20T21:42:18.060 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 407685120 bytes 2026-02-20T21:42:18.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 439476224 bytes 2026-02-20T21:42:18.816 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 446513152 bytes 2026-02-20T21:42:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 510138368 bytes 2026-02-20T21:42:19.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 510740480 bytes 2026-02-20T21:42:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 561950720 bytes 2026-02-20T21:42:21.108 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 636184576 bytes 2026-02-20T21:42:21.267 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 636821504 bytes 2026-02-20T21:42:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 695972864 bytes 2026-02-20T21:42:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 773237760 bytes 2026-02-20T21:42:23.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2200 IOs, 797919232 bytes 2026-02-20T21:42:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:42:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:42:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:42:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:42:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_5 present 2026-02-20T21:42:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:42:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:42:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-20T21:42:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:42:23.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:42:23.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:42:23.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_5 2026-02-20T21:42:23.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:42:23.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:42:23.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:42:23.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-20T21:42:23.674 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:42:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=1212686c2488 2026-02-20T21:42:23.707 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:42:23.708 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:42:23.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_5 present 1212686c2488 2026-02-20T21:42:23.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:42:23.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:23.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:42:23.809 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:42:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=1212686c2488 2026-02-20T21:42:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:42:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:42:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_5 2026-02-20T21:42:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:42:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:42:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:42:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-20T21:42:23.810 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:42:23.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=1212686c2488 2026-02-20T21:42:23.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1212686c2488 2026-02-20T21:42:23.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 1212686c2488 -o 1212686c2488 = 1212686c2488 2026-02-20T21:42:23.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-20T21:42:23.841 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-20T21:42:23.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:42:23.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_5 2026-02-20T21:42:23.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:23.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:23.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:42:23.842 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_5 2026-02-20T21:42:37.652 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.9916088808141523) > weight (0.25) 2026-02-20T21:42:37.652 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.5990291618954506) > weight (0.25) 2026-02-20T21:42:37.652 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.6178674282192649) > weight (0.25) 2026-02-20T21:42:37.652 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-20T21:42:37.653 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-20T21:42:37.653 INFO:tasks.rbd_mirror_thrash:waiting for 5.417677043408525 secs before reviving daemons 2026-02-20T21:42:37.654 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:42:37.651+0000 7ffa09414640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 126480) UID: 1000 2026-02-20T21:42:43.072 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-20T21:42:43.072 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:42:43.072 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-20T21:42:43.073 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-20T21:42:43.073 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-20T21:42:43.076 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-20T21:42:43.076 INFO:tasks.rbd_mirror_thrash:waiting for 87.836735357088 secs before thrashing 2026-02-20T21:42:43.639 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.9946463279981139) > weight (0.25) 2026-02-20T21:42:43.639 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.1 2026-02-20T21:42:43.639 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Sent signal 15 2026-02-20T21:42:43.639 INFO:tasks.rbd_mirror_thrash:waiting for 8.830949315177294 secs before reviving daemons 2026-02-20T21:42:43.640 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:42:43.638+0000 7ff52ae4c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 99850) UID: 1000 2026-02-20T21:42:52.471 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.1 2026-02-20T21:42:52.472 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:42:52.472 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.1 2026-02-20T21:42:52.472 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Restarting daemon 2026-02-20T21:42:52.472 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 2026-02-20T21:42:52.475 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Started 2026-02-20T21:42:52.475 INFO:tasks.rbd_mirror_thrash:waiting for 116.73733401155641 secs before thrashing 2026-02-20T21:42:53.186 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-20T21:42:53.192 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_5 deleted 2026-02-20T21:42:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:42:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:42:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:42:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:42:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:42:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:42:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_5 2026-02-20T21:42:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:42:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:42:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:42:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-20T21:42:53.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:42:53.217 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-20T21:42:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:42:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:42:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:42:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_5 deleted '' 2026-02-20T21:42:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:42:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:42:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:42:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:42:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:42:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:42:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_5 2026-02-20T21:42:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:42:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:42:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_5 2026-02-20T21:42:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_5 2026-02-20T21:42:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:42:53.342 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_5: (2) No such file or directory 2026-02-20T21:42:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:42:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:42:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:42:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:42:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_5 2026-02-20T21:42:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:42:53.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_5 2026-02-20T21:42:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_5 2026-02-20T21:42:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:42:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:42:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:42:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_5 2026-02-20T21:42:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_5 2026-02-20T21:42:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_5 2026-02-20T21:42:53.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_5 2026-02-20T21:42:53.511 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:42:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:42:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:42:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_6 2026-02-20T21:42:53.514 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_6 512M 2026-02-20T21:42:53.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:53.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:42:53.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:53.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:42:53.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:42:53.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:42:53.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:42:53.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-20T21:42:53.515 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-20T21:42:53.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:42:53.516 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_6 2026-02-20T21:42:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_6 2026-02-20T21:42:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:42:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:42:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_6 journal 2026-02-20T21:42:53.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-20T21:42:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_6 2026-02-20T21:42:53.613 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_6': 2026-02-20T21:42:53.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:42:53.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:42:53.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-20T21:42:53.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 30dba1e21c35 2026-02-20T21:42:53.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.30dba1e21c35 2026-02-20T21:42:53.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:42:53.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:42:53.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:42:53.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:42:53.614 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:42:53 2026 2026-02-20T21:42:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:42:53 2026 2026-02-20T21:42:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:42:53 2026 2026-02-20T21:42:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 30dba1e21c35 2026-02-20T21:42:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:42:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:42:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 28ffac6e-3ae6-4832-9eb4-bd6b24e53ae6 2026-02-20T21:42:53.615 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-20T21:42:53.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_6 2026-02-20T21:42:53.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:42:53.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:42:53.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:42:53.616 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:42:53.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-20T21:42:53.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:42:53.617 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_6 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:42:54.436 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 64498688 bytes 2026-02-20T21:42:55.457 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 131200000 bytes 2026-02-20T21:42:55.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 180892672 bytes 2026-02-20T21:42:57.120 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 262857728 bytes 2026-02-20T21:42:57.592 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 277661696 bytes 2026-02-20T21:42:57.969 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 297421824 bytes 2026-02-20T21:42:58.595 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 321338368 bytes 2026-02-20T21:42:59.185 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 381686784 bytes 2026-02-20T21:42:59.704 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 438772736 bytes 2026-02-20T21:43:00.542 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 471694336 bytes 2026-02-20T21:43:00.812 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 500144128 bytes 2026-02-20T21:43:01.297 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 516150272 bytes 2026-02-20T21:43:01.719 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 561362944 bytes 2026-02-20T21:43:03.021 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 651323392 bytes 2026-02-20T21:43:03.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 674671616 bytes 2026-02-20T21:43:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 722405376 bytes 2026-02-20T21:43:04.964 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 795752448 bytes 2026-02-20T21:43:05.797 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 847753216 bytes 2026-02-20T21:43:07.471 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 972121088 bytes 2026-02-20T21:43:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:43:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:43:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:43:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:43:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_6 present 2026-02-20T21:43:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:43:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:43:07.626 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-20T21:43:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:43:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:43:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:43:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_6 2026-02-20T21:43:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:43:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:43:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:43:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-20T21:43:07.627 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:43:07.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=1212b8f92ff1 2026-02-20T21:43:07.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:43:07.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:43:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_6 present 1212b8f92ff1 2026-02-20T21:43:07.759 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:43:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:43:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:43:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=1212b8f92ff1 2026-02-20T21:43:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:43:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:43:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_6 2026-02-20T21:43:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:43:07.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:43:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:43:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-20T21:43:07.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:43:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=1212b8f92ff1 2026-02-20T21:43:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1212b8f92ff1 2026-02-20T21:43:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 1212b8f92ff1 -o 1212b8f92ff1 = 1212b8f92ff1 2026-02-20T21:43:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-20T21:43:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-20T21:43:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:43:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_6 2026-02-20T21:43:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:43:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:43:07.788 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_6 2026-02-20T21:43:36.230 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-20T21:43:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_6 deleted 2026-02-20T21:43:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:43:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:36.236 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:43:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:43:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:43:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:43:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:43:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_6 2026-02-20T21:43:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:43:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:43:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:43:36.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-20T21:43:36.238 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:43:36.261 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-20T21:43:36.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:43:36.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:43:36.263 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:43:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_6 deleted '' 2026-02-20T21:43:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:43:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:43:36.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:43:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:43:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:43:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:43:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_6 2026-02-20T21:43:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:43:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:43:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_6 2026-02-20T21:43:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_6 2026-02-20T21:43:36.366 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:43:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_6: (2) No such file or directory 2026-02-20T21:43:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:43:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:43:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:43:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:43:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_6 2026-02-20T21:43:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:43:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:43:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_6 2026-02-20T21:43:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_6 2026-02-20T21:43:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:43:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:43:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:43:36.421 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:43:36.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_6 2026-02-20T21:43:36.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:43:36.422 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_6 2026-02-20T21:43:36.423 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_6 2026-02-20T21:43:36.449 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_6 2026-02-20T21:43:36.560 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:43:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:43:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:43:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_7 2026-02-20T21:43:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_7 512M 2026-02-20T21:43:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:43:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:43:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:43:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:43:36.562 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-20T21:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-20T21:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:43:36.563 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_7 2026-02-20T21:43:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_7 2026-02-20T21:43:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:43:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:43:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:43:36.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_7 journal 2026-02-20T21:43:36.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-20T21:43:36.632 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_7 2026-02-20T21:43:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_7': 2026-02-20T21:43:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:43:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:43:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-20T21:43:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 30f378a1874 2026-02-20T21:43:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.30f378a1874 2026-02-20T21:43:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:43:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:43:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:43:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:43:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:43:36 2026 2026-02-20T21:43:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:43:36 2026 2026-02-20T21:43:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:43:36 2026 2026-02-20T21:43:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 30f378a1874 2026-02-20T21:43:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:43:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:43:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 78c1d009-2f1d-40ba-9e3a-b6e81e9a1298 2026-02-20T21:43:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-20T21:43:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_7 2026-02-20T21:43:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:43:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:43:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:43:36.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-20T21:43:36.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:43:36.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_7 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:43:37.356 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 55418880 bytes 2026-02-20T21:43:38.412 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 111298560 bytes 2026-02-20T21:43:39.452 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 180016128 bytes 2026-02-20T21:43:40.068 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 204667904 bytes 2026-02-20T21:43:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 261008384 bytes 2026-02-20T21:43:41.750 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 300720128 bytes 2026-02-20T21:43:42.056 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 325069824 bytes 2026-02-20T21:43:43.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 365392896 bytes 2026-02-20T21:43:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:43:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:43:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:43:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:43:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_7 present 2026-02-20T21:43:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:43:43.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:43:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-20T21:43:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:43:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:43:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:43:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_7 2026-02-20T21:43:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:43:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:43:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:43:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-20T21:43:43.669 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:43:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=1212873d42fb 2026-02-20T21:43:43.697 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:43:43.698 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:43:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_7 present 1212873d42fb 2026-02-20T21:43:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:43:43.799 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:43:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:43:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=1212873d42fb 2026-02-20T21:43:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:43:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:43:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_7 2026-02-20T21:43:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:43:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:43:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:43:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-20T21:43:43.800 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:43:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=1212873d42fb 2026-02-20T21:43:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1212873d42fb 2026-02-20T21:43:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 1212873d42fb -o 1212873d42fb = 1212873d42fb 2026-02-20T21:43:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-20T21:43:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-20T21:43:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:43:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_7 2026-02-20T21:43:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:43:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:43:43.830 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:43:43.831 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_7 2026-02-20T21:44:10.913 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.0 2026-02-20T21:44:10.914 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Sent signal 15 2026-02-20T21:44:10.914 INFO:tasks.rbd_mirror_thrash:waiting for 3.0949441397999857 secs before reviving daemons 2026-02-20T21:44:10.915 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:44:10.913+0000 7fd69aa50640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 111351) UID: 1000 2026-02-20T21:44:14.010 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.0 2026-02-20T21:44:14.010 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:44:14.011 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.0 2026-02-20T21:44:14.011 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Restarting daemon 2026-02-20T21:44:14.011 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 2026-02-20T21:44:14.014 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Started 2026-02-20T21:44:14.014 INFO:tasks.rbd_mirror_thrash:waiting for 93.62862275623465 secs before thrashing 2026-02-20T21:44:14.330 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-20T21:44:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_7 deleted 2026-02-20T21:44:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:44:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:44:14.337 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:44:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:44:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:44:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:44:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_7 2026-02-20T21:44:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:44:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:44:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:44:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-20T21:44:14.338 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:44:14.360 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-20T21:44:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:44:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:44:14.362 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:44:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_7 deleted '' 2026-02-20T21:44:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:44:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:44:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:44:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:44:14.464 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:44:14.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:44:14.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_7 2026-02-20T21:44:14.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:44:14.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:44:14.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_7 2026-02-20T21:44:14.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_7 2026-02-20T21:44:14.465 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:44:14.485 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_7: (2) No such file or directory 2026-02-20T21:44:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:44:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:44:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:44:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:44:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_7 2026-02-20T21:44:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:44:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_7 2026-02-20T21:44:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_7 2026-02-20T21:44:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:44:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:44:14.517 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:44:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_7 2026-02-20T21:44:14.518 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:14.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:14.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_7 2026-02-20T21:44:14.519 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_7 2026-02-20T21:44:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_7 2026-02-20T21:44:14.655 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:44:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:44:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:44:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_8 2026-02-20T21:44:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_8 512M 2026-02-20T21:44:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:44:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:44:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:44:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:44:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:44:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-20T21:44:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-20T21:44:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:44:14.662 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_8 2026-02-20T21:44:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_8 2026-02-20T21:44:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:44:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:44:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_8 journal 2026-02-20T21:44:14.726 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-20T21:44:14.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_8 2026-02-20T21:44:14.753 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_8': 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 3110414b170e 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.3110414b170e 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:44:14 2026 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:44:14 2026 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:44:14 2026 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 3110414b170e 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 45a08cdd-7a03-437a-8c9f-69ab6978d8ee 2026-02-20T21:44:14.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-20T21:44:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_8 2026-02-20T21:44:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:14.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:44:14.756 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:44:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=12 2026-02-20T21:44:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:44:14.757 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 12s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_8 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:44:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 77498368 bytes 2026-02-20T21:44:16.717 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 127163392 bytes 2026-02-20T21:44:17.419 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 155885568 bytes 2026-02-20T21:44:18.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 233338880 bytes 2026-02-20T21:44:18.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 236345344 bytes 2026-02-20T21:44:18.811 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 242134016 bytes 2026-02-20T21:44:19.194 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 275118080 bytes 2026-02-20T21:44:19.774 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 304060416 bytes 2026-02-20T21:44:20.540 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 352105472 bytes 2026-02-20T21:44:20.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 361558016 bytes 2026-02-20T21:44:21.365 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 408639488 bytes 2026-02-20T21:44:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 410442752 bytes 2026-02-20T21:44:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 415656960 bytes 2026-02-20T21:44:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 474668032 bytes 2026-02-20T21:44:23.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 537154560 bytes 2026-02-20T21:44:24.304 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 563132416 bytes 2026-02-20T21:44:24.387 INFO:tasks.ceph.cluster2.osd.2.trial163.stderr:2026-02-20T21:44:24.385+0000 7fdf668cb640 -1 reset not still connected to 0x55a640abc410 2026-02-20T21:44:24.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 565404672 bytes 2026-02-20T21:44:24.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1800 IOs, 575970304 bytes 2026-02-20T21:44:24.729 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1900 IOs, 576576512 bytes 2026-02-20T21:44:25.278 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2000 IOs, 619542528 bytes 2026-02-20T21:44:25.369 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2100 IOs, 638713856 bytes 2026-02-20T21:44:26.319 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:2200 IOs, 689331200 bytes 2026-02-20T21:44:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:44:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:44:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:44:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:44:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_8 present 2026-02-20T21:44:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:44:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:26.765 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:44:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-20T21:44:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:44:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:44:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:44:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_8 2026-02-20T21:44:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:44:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:44:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:44:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-20T21:44:26.766 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:44:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=1212eb6225df 2026-02-20T21:44:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:44:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:44:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_8 present 1212eb6225df 2026-02-20T21:44:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:44:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:44:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:44:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=1212eb6225df 2026-02-20T21:44:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:44:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:44:27.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_8 2026-02-20T21:44:27.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:44:27.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:44:27.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:44:27.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-20T21:44:27.104 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:44:27.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=1212eb6225df 2026-02-20T21:44:27.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1212eb6225df 2026-02-20T21:44:27.133 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 1212eb6225df -o 1212eb6225df = 1212eb6225df 2026-02-20T21:44:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-20T21:44:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-20T21:44:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:44:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_8 2026-02-20T21:44:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:44:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_8 2026-02-20T21:44:49.213 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.0 with skip (0.5610214653692391) > weight (0.25) 2026-02-20T21:44:49.214 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster2.client.mirror.1 with skip (0.85874072998692) > weight (0.25) 2026-02-20T21:44:49.214 INFO:tasks.rbd_mirror_thrash:kill cluster2.client.mirror.2 2026-02-20T21:44:49.214 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Sent signal 15 2026-02-20T21:44:49.214 INFO:tasks.rbd_mirror_thrash:waiting for 5.75523566515213 secs before reviving daemons 2026-02-20T21:44:49.215 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:44:49.213+0000 7fbd1dadb640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 129206) UID: 1000 2026-02-20T21:44:54.970 INFO:tasks.rbd_mirror_thrash:waiting for cluster2.client.mirror.2 2026-02-20T21:44:54.970 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:44:54.971 INFO:tasks.rbd_mirror_thrash:reviving cluster2.client.mirror.2 2026-02-20T21:44:54.971 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Restarting daemon 2026-02-20T21:44:54.971 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 2026-02-20T21:44:54.974 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Started 2026-02-20T21:44:54.974 INFO:tasks.rbd_mirror_thrash:waiting for 111.95616901013395 secs before thrashing 2026-02-20T21:44:57.219 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-20T21:44:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_8 deleted 2026-02-20T21:44:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:44:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:44:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:44:57.224 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:44:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:44:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:44:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_8 2026-02-20T21:44:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:44:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:44:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:44:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-20T21:44:57.225 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:44:57.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-20T21:44:57.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:44:57.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:44:57.251 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:44:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_8 deleted '' 2026-02-20T21:44:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:44:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:44:57.353 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:44:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:44:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:44:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:44:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_8 2026-02-20T21:44:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:44:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:44:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_8 2026-02-20T21:44:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_8 2026-02-20T21:44:57.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:44:57.377 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_8: (2) No such file or directory 2026-02-20T21:44:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:44:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:44:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:44:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:44:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_8 2026-02-20T21:44:57.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:57.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:57.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:44:57.380 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_8 2026-02-20T21:44:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_8 2026-02-20T21:44:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:44:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:44:57.408 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:44:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_8 2026-02-20T21:44:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:57.409 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_8 2026-02-20T21:44:57.410 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_8 2026-02-20T21:44:57.642 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_8 2026-02-20T21:44:57.751 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:44:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:44:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:44:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_9 2026-02-20T21:44:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_9 512M 2026-02-20T21:44:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:44:57.754 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:57.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:44:57.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:44:57.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:44:57.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:44:57.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-20T21:44:57.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-20T21:44:57.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:44:57.755 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_9 2026-02-20T21:44:57.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_9 2026-02-20T21:44:57.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:57.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:57.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:44:57.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:44:57.792 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_9 journal 2026-02-20T21:44:57.824 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-20T21:44:57.827 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_9 2026-02-20T21:44:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_9': 2026-02-20T21:44:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:44:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:44:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-20T21:44:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 3132748ecb66 2026-02-20T21:44:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.3132748ecb66 2026-02-20T21:44:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:44:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:44:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:44:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:44:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:44:57 2026 2026-02-20T21:44:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:44:57 2026 2026-02-20T21:44:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:44:57 2026 2026-02-20T21:44:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 3132748ecb66 2026-02-20T21:44:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:44:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:44:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: c67965a9-6dbb-4335-baec-9d7b81f10424 2026-02-20T21:44:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-20T21:44:57.854 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_9 2026-02-20T21:44:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:44:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:44:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:44:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:44:57.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=7 2026-02-20T21:44:57.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:44:57.856 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 7s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_9 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:44:58.896 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 45066240 bytes 2026-02-20T21:44:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 92517376 bytes 2026-02-20T21:45:00.428 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 156500992 bytes 2026-02-20T21:45:01.041 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 161978368 bytes 2026-02-20T21:45:01.568 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 190476288 bytes 2026-02-20T21:45:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 258987008 bytes 2026-02-20T21:45:03.483 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 351890432 bytes 2026-02-20T21:45:03.578 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 354481152 bytes 2026-02-20T21:45:03.777 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 365554688 bytes 2026-02-20T21:45:04.035 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 379718656 bytes 2026-02-20T21:45:04.099 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 404084736 bytes 2026-02-20T21:45:04.820 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 425641984 bytes 2026-02-20T21:45:04.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:45:04.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:45:04.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:45:04.866 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:45:04.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_9 present 2026-02-20T21:45:04.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:45:04.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:04.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:45:04.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-20T21:45:04.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:45:04.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:45:04.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:45:04.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_9 2026-02-20T21:45:04.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:45:04.867 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:45:04.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:45:04.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-20T21:45:04.868 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:45:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=12121216d1cc 2026-02-20T21:45:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:45:04.897 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:45:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_9 present 12121216d1cc 2026-02-20T21:45:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:45:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:45:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:45:04.999 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=12121216d1cc 2026-02-20T21:45:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:45:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:45:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_9 2026-02-20T21:45:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:45:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:45:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:45:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-20T21:45:05.000 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:45:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=12121216d1cc 2026-02-20T21:45:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 12121216d1cc 2026-02-20T21:45:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 12121216d1cc -o 12121216d1cc = 12121216d1cc 2026-02-20T21:45:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-20T21:45:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-20T21:45:05.030 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:45:05.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_9 2026-02-20T21:45:05.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:45:05.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:05.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:45:05.031 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_9 2026-02-20T21:45:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-20T21:45:35.163 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_9 deleted 2026-02-20T21:45:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:45:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:45:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:45:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:45:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:45:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:45:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_9 2026-02-20T21:45:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:45:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:45:35.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:45:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-20T21:45:35.165 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:45:35.187 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-20T21:45:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:45:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:45:35.189 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:45:35.291 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_9 deleted '' 2026-02-20T21:45:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:45:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:45:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:45:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:45:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:45:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:45:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_9 2026-02-20T21:45:35.292 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:45:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:45:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_9 2026-02-20T21:45:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_9 2026-02-20T21:45:35.293 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:45:35.313 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_9: (2) No such file or directory 2026-02-20T21:45:35.315 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:45:35.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:45:35.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:45:35.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:45:35.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_9 2026-02-20T21:45:35.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:45:35.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:35.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:45:35.316 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_9 2026-02-20T21:45:35.345 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_9 2026-02-20T21:45:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:45:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:45:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:45:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:45:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_9 2026-02-20T21:45:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:45:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_9 2026-02-20T21:45:35.346 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_9 2026-02-20T21:45:35.374 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_9 2026-02-20T21:45:35.484 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:45:35.487 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:45:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for i in `seq 1 10` 2026-02-20T21:45:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image=image_10 2026-02-20T21:45:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ create_image cluster2 mirror image_10 512M 2026-02-20T21:45:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:45:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:45:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:45:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:45:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:45:35.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local size=128 2026-02-20T21:45:35.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -n 512M ']' 2026-02-20T21:45:35.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ size=512M 2026-02-20T21:45:35.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ shift 2026-02-20T21:45:35.489 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 create --size 512M --image-feature layering,exclusive-lock,journaling mirror/image_10 2026-02-20T21:45:35.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ enable_mirror cluster2 mirror image_10 2026-02-20T21:45:35.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:45:35.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:35.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:45:35.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local mode=journal 2026-02-20T21:45:35.525 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/image_10 journal 2026-02-20T21:45:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring enabled 2026-02-20T21:45:35.559 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 info mirror/image_10 2026-02-20T21:45:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:rbd image 'image_10': 2026-02-20T21:45:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: size 512 MiB in 128 objects 2026-02-20T21:45:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: order 22 (4 MiB objects) 2026-02-20T21:45:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: snapshot_count: 0 2026-02-20T21:45:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: id: 314a18de11a2 2026-02-20T21:45:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: block_name_prefix: rbd_data.314a18de11a2 2026-02-20T21:45:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: format: 2 2026-02-20T21:45:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: features: layering, exclusive-lock, journaling 2026-02-20T21:45:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: op_features: 2026-02-20T21:45:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: flags: 2026-02-20T21:45:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: create_timestamp: Fri Feb 20 21:45:35 2026 2026-02-20T21:45:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: access_timestamp: Fri Feb 20 21:45:35 2026 2026-02-20T21:45:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: modify_timestamp: Fri Feb 20 21:45:35 2026 2026-02-20T21:45:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: journal: 314a18de11a2 2026-02-20T21:45:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring state: enabled 2026-02-20T21:45:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring mode: journal 2026-02-20T21:45:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring global id: 216f50a2-2302-4e92-9ff4-97b576c0cc57 2026-02-20T21:45:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout: mirroring primary: true 2026-02-20T21:45:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stress_write_image cluster2 mirror image_10 2026-02-20T21:45:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:45:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:45:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ awk 'BEGIN {srand(); print int(10 * rand()) + 5}' 2026-02-20T21:45:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local duration=14 2026-02-20T21:45:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:45:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ timeout 14s ceph_test_rbd_mirror_random_write --cluster cluster2 mirror image_10 --debug-rbd=20 --debug-journaler=20 2026-02-20T21:45:36.546 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:100 IOs, 55391232 bytes 2026-02-20T21:45:37.213 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:200 IOs, 102507520 bytes 2026-02-20T21:45:37.354 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:300 IOs, 109007872 bytes 2026-02-20T21:45:38.095 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:400 IOs, 130373632 bytes 2026-02-20T21:45:38.534 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:500 IOs, 176390144 bytes 2026-02-20T21:45:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:600 IOs, 193831936 bytes 2026-02-20T21:45:40.695 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:700 IOs, 321945600 bytes 2026-02-20T21:45:41.397 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:800 IOs, 432473088 bytes 2026-02-20T21:45:42.025 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:900 IOs, 441297920 bytes 2026-02-20T21:45:42.392 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1000 IOs, 459189248 bytes 2026-02-20T21:45:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1100 IOs, 537668608 bytes 2026-02-20T21:45:44.654 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1200 IOs, 587847680 bytes 2026-02-20T21:45:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1300 IOs, 643133440 bytes 2026-02-20T21:45:46.544 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1400 IOs, 718893056 bytes 2026-02-20T21:45:47.216 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1500 IOs, 751751168 bytes 2026-02-20T21:45:47.644 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.0 with skip (0.5607484420041222) > weight (0.25) 2026-02-20T21:45:47.644 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.1 with skip (0.6663890222073973) > weight (0.25) 2026-02-20T21:45:47.644 INFO:tasks.rbd_mirror_thrash:skipping daemon cluster1.client.mirror.2 with skip (0.5317194084936497) > weight (0.25) 2026-02-20T21:45:47.644 INFO:tasks.rbd_mirror_thrash:kill cluster1.client.mirror.3 2026-02-20T21:45:47.645 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Sent signal 15 2026-02-20T21:45:47.645 INFO:tasks.rbd_mirror_thrash:waiting for 6.832285344757412 secs before reviving daemons 2026-02-20T21:45:47.646 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:45:47.644+0000 7f4d5871a640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 140413) UID: 1000 2026-02-20T21:45:47.678 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1600 IOs, 805788672 bytes 2026-02-20T21:45:49.135 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:1700 IOs, 889056256 bytes 2026-02-20T21:45:49.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=124 2026-02-20T21:45:49.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:45:49.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 124 -eq 124 ']' 2026-02-20T21:45:49.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:45:49.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_10 present 2026-02-20T21:45:49.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:45:49.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:49.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:45:49.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=present 2026-02-20T21:45:49.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:45:49.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:45:49.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:45:49.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_10 2026-02-20T21:45:49.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:45:49.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:45:49.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-20T21:45:49.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-20T21:45:49.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:45:49.629 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id=1212ec2d9bdc 2026-02-20T21:45:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:45:49.630 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:45:49.731 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_10 present 1212ec2d9bdc 2026-02-20T21:45:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:45:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:45:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=present 2026-02-20T21:45:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id=1212ec2d9bdc 2026-02-20T21:45:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:45:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:45:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_10 2026-02-20T21:45:49.732 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:45:49.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:45:49.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-20T21:45:49.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-20T21:45:49.733 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:45:49.760 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id=1212ec2d9bdc 2026-02-20T21:45:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1212ec2d9bdc 2026-02-20T21:45:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -z 1212ec2d9bdc -o 1212ec2d9bdc = 1212ec2d9bdc 2026-02-20T21:45:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_state=present 2026-02-20T21:45:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test present = present 2026-02-20T21:45:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:45:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ disable_mirror cluster2 mirror image_10 2026-02-20T21:45:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:45:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:45:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:45:49.761 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/image_10 2026-02-20T21:45:54.478 INFO:tasks.rbd_mirror_thrash:waiting for cluster1.client.mirror.3 2026-02-20T21:45:54.478 INFO:teuthology.orchestra.run:waiting for 600 2026-02-20T21:45:54.479 INFO:tasks.rbd_mirror_thrash:reviving cluster1.client.mirror.3 2026-02-20T21:45:54.479 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Restarting daemon 2026-02-20T21:45:54.479 DEBUG:teuthology.orchestra.run.trial163:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 2026-02-20T21:45:54.482 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Started 2026-02-20T21:45:54.482 INFO:tasks.rbd_mirror_thrash:waiting for 115.25143694057569 secs before thrashing 2026-02-20T21:46:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:Mirroring disabled 2026-02-20T21:46:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_present cluster1 mirror image_10 deleted 2026-02-20T21:46:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:46:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:46:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local state=deleted 2026-02-20T21:46:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:46:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local s 2026-02-20T21:46:20.249 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:46:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_10 2026-02-20T21:46:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:46:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-20T21:46:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-20T21:46:20.250 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:46:20.274 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-20T21:46:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ image_id= 2026-02-20T21:46:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-20T21:46:20.276 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0.1 2026-02-20T21:46:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test_image_present cluster1 mirror image_10 deleted '' 2026-02-20T21:46:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:46:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:46:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local test_state=deleted 2026-02-20T21:46:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image_id= 2026-02-20T21:46:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_state=deleted 2026-02-20T21:46:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local current_image_id 2026-02-20T21:46:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ get_image_id cluster1 mirror image_10 2026-02-20T21:46:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:46:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local image=image_10 2026-02-20T21:46:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rbd --cluster=cluster1 info mirror/image_10 2026-02-20T21:46:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-20T21:46:20.401 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:rbd: error opening image image_10: (2) No such file or directory 2026-02-20T21:46:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ current_image_id= 2026-02-20T21:46:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n '' 2026-02-20T21:46:20.403 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test deleted = deleted 2026-02-20T21:46:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:46:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ purge_snapshots cluster2 mirror image_10 2026-02-20T21:46:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:46:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:46:20.404 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster cluster2 snap purge mirror/image_10 2026-02-20T21:46:20.432 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image_retry cluster2 mirror image_10 2026-02-20T21:46:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:46:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:46:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-20T21:46:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:46:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ remove_image cluster2 mirror image_10 2026-02-20T21:46:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:46:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local image=image_10 2026-02-20T21:46:20.433 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 snap purge mirror/image_10 2026-02-20T21:46:20.461 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rbd --cluster=cluster2 rm mirror/image_10 2026-02-20T21:46:20.567 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-20T21:46:20.569 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:46:20.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2026-02-20T21:46:20.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ tee -a /tmp/tmp.IQNOxgiLiE/rbd-mirror.test.log 2026-02-20T21:46:20.570 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ date '+%F %T' 2026-02-20T21:46:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo 2026-02-20 21:46:20 TEST: check if removed 'images'\''' OMAP are removed 2026-02-20T21:46:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-20 21:46:20 TEST: check if removed images' OMAP are removed 2026-02-20T21:46:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-20T21:46:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:46:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-20T21:46:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:46:20.572 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local obj_name=rbd_mirroring 2026-02-20T21:46:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local filter=status_global 2026-02-20T21:46:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:46:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:46:20.573 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:46:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-20T21:46:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:46:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:46:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local filter=status_global 2026-02-20T21:46:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-20T21:46:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:46:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:46:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-20T21:46:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -c status_global 2026-02-20T21:46:20.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-20T21:46:20.599 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-20T21:46:20.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:46:20.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:46:20.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:46:20.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-20T21:46:20.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:46:20.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local obj_name=rbd_mirroring 2026-02-20T21:46:20.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local filter=image_ 2026-02-20T21:46:20.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:46:20.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:46:20.600 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:46:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-20T21:46:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:46:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:46:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local filter=image_ 2026-02-20T21:46:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-20T21:46:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:46:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:46:20.602 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-20T21:46:20.603 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -c image_ 2026-02-20T21:46:20.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-20T21:46:20.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-20T21:46:20.622 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:46:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:46:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:46:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-20T21:46:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:46:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local obj_name=rbd_mirror_leader 2026-02-20T21:46:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local filter=image_map 2026-02-20T21:46:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:46:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:46:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:46:20.623 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-20T21:46:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:46:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:46:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local filter=image_map 2026-02-20T21:46:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-20T21:46:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster1 2026-02-20T21:46:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -c image_map 2026-02-20T21:46:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:46:20.624 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T21:46:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-20T21:46:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-20T21:46:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:46:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:46:20.646 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local obj_name=rbd_mirroring 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local filter=status_global 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:46:20.647 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local filter=status_global 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-20T21:46:20.648 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -c status_global 2026-02-20T21:46:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-20T21:46:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-20T21:46:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:46:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:46:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:46:20.665 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-20T21:46:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:46:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local obj_name=rbd_mirroring 2026-02-20T21:46:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local filter=image_ 2026-02-20T21:46:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:46:20.666 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:46:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:46:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-20T21:46:20.667 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:46:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:46:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local filter=image_ 2026-02-20T21:46:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-20T21:46:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:46:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirroring 2026-02-20T21:46:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -c image_ 2026-02-20T21:46:20.668 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-20T21:46:20.687 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-20T21:46:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-20T21:46:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:46:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:46:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:46:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-20T21:46:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:46:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local pool=mirror 2026-02-20T21:46:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local obj_name=rbd_mirror_leader 2026-02-20T21:46:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local filter=image_map 2026-02-20T21:46:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-20T21:46:20.688 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ sleep 0 2026-02-20T21:46:20.689 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:46:20.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-20T21:46:20.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:46:20.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:46:20.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local filter=image_map 2026-02-20T21:46:20.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-20T21:46:20.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ grep -c image_map 2026-02-20T21:46:20.690 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local cluster=cluster2 2026-02-20T21:46:20.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local pool=mirror 2026-02-20T21:46:20.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ local obj_name=rbd_mirror_leader 2026-02-20T21:46:20.691 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-20T21:46:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test 0 = 0 2026-02-20T21:46:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ error_code=0 2026-02-20T21:46:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set -e 2026-02-20T21:46:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:46:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 0 2026-02-20T21:46:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ cleanup 0 2026-02-20T21:46:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local error_code=0 2026-02-20T21:46:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ set +e 2026-02-20T21:46:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -ne 0 ']' 2026-02-20T21:46:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z '' ']' 2026-02-20T21:46:20.709 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster instance 2026-02-20T21:46:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:46:20.710 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-20T21:46:20.963 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:2026-02-20T21:46:20.961+0000 7fe80a575640 0 --2- 10.20.193.163:0/4286332592 >> [v2:10.20.193.69:3300/0,v1:10.20.193.69:6789/0] conn(0x7fe80c05a490 0x7fe80c106030 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2026-02-20T21:46:22.107 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror' does not exist 2026-02-20T21:46:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:46:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-20T21:46:22.569 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:22.566+0000 7f0e45db3640 -1 rbd::mirror::RemotePoolPoller: 0x5624793f2dd0 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2026-02-20T21:46:22.570 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:22.566+0000 7f0e465b4640 -1 rbd::mirror::RemotePoolPoller: 0x5624793f2dd0 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2026-02-20T21:46:22.570 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:22.566+0000 7f0e465b4640 -1 rbd::mirror::PoolReplayer: 0x56247a210000 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2026-02-20T21:46:22.576 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:22.573+0000 7f0e45db3640 -1 rbd::mirror::MirrorStatusUpdater 0x56247a4da360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:22.576 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:22.573+0000 7f0e535ce640 -1 rbd::mirror::NamespaceReplayer: 0x56247a4dc1c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:22.577 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:22.574+0000 7f0e465b4640 -1 rbd::mirror::MirrorStatusUpdater 0x56247a4da480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:22.577 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:22.574+0000 7f0e465b4640 -1 rbd::mirror::MirrorStatusUpdater 0x56247a4da5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:22.577 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:22.574+0000 7f0e535ce640 -1 rbd::mirror::NamespaceReplayer: 0x56247a4dc000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:22.577 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:22.574+0000 7f0e535ce640 -1 rbd::mirror::NamespaceReplayer: 0x562479406e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:22.606 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:22.604+0000 7f0e56e2fa40 -1 rbd::mirror::Mirror: 0x562479462240 update_pool_replayers: restarting failed pool replayer for uuid: b19e0387-9a8c-4ce0-9037-ecde016f05b9 cluster: cluster1 client: client.mirror 2026-02-20T21:46:22.624 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:22.621+0000 7f0e56e2fa40 -1 rbd::mirror::PoolReplayer: 0x56247a210000 init: error accessing remote pool mirror: (2) No such file or directory 2026-02-20T21:46:23.161 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror' does not exist 2026-02-20T21:46:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:46:23.171 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-20T21:46:24.109 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror_parent' does not exist 2026-02-20T21:46:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ CEPH_ARGS= 2026-02-20T21:46:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-20T21:46:24.358 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.354+0000 7f8f7743a640 -1 rbd::mirror::LeaderWatcher: 0x562e8fb1a300 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-02-20T21:46:24.358 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.354+0000 7f8f7142e640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x562e9037d440 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T21:46:24.358 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.354+0000 7f8f7142e640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x562e90a098c0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T21:46:24.358 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.354+0000 7f8f7142e640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x562e9037cb40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T21:46:24.358 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.354+0000 7f8f70c2d640 -1 rbd::mirror::PoolWatcher: 0x562e8fc71860 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:24.358 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.354+0000 7f8f7142e640 -1 rbd::mirror::PoolWatcher: 0x562e8fc71680 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:24.358 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.354+0000 7f8f6a420640 -1 rbd::mirror::PoolWatcher: 0x562e906ecb40 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:24.358 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.354+0000 7f8f6a420640 -1 rbd::mirror::PoolWatcher: 0x562e906ecd20 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:24.359 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.354+0000 7f8f6a420640 -1 rbd::mirror::PoolWatcher: 0x562e906ec960 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:24.359 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.354+0000 7f8f7743a640 -1 rbd::mirror::LeaderWatcher: 0x562e8fb1a300 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-20T21:46:24.359 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.354+0000 7f8f7743a640 -1 rbd::mirror::LeaderWatcher: 0x562e8fb1a300 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-02-20T21:46:24.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.559+0000 7f23fd244640 -1 rbd::mirror::RemotePoolPoller: 0x5625a2ef8dd0 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2026-02-20T21:46:24.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.559+0000 7f23fca43640 -1 rbd::mirror::RemotePoolPoller: 0x5625a2ef8dd0 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2026-02-20T21:46:24.561 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.559+0000 7f23fca43640 -1 rbd::mirror::PoolReplayer: 0x5625a3d06000 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2026-02-20T21:46:24.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.559+0000 7f2409a5d640 -1 rbd::mirror::InstanceWatcher: 0x5625a49c7180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5479 object: (2) No such file or directory 2026-02-20T21:46:24.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.559+0000 7f240a25e640 -1 rbd::mirror::InstanceWatcher: 0x5625a4dbe000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5479 object: (2) No such file or directory 2026-02-20T21:46:24.562 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.559+0000 7f240a25e640 -1 rbd::mirror::InstanceWatcher: 0x5625a4dbe380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5479 object: (2) No such file or directory 2026-02-20T21:46:24.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.560+0000 7f2403250640 -1 rbd::mirror::InstanceWatcher: 0x5625a49c7180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:24.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.560+0000 7f23fd244640 -1 rbd::mirror::MirrorStatusUpdater 0x5625a3fe65a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:24.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.560+0000 7f240a25e640 -1 rbd::mirror::NamespaceReplayer: 0x5625a2f0c700 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:24.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.560+0000 7f2403a51640 -1 rbd::mirror::InstanceWatcher: 0x5625a4dbe000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:24.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.560+0000 7f2403250640 -1 rbd::mirror::InstanceWatcher: 0x5625a4dbe380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:24.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.560+0000 7f2403a51640 -1 rbd::mirror::MirrorStatusUpdater 0x5625a3fe6000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:24.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.560+0000 7f2409a5d640 -1 rbd::mirror::NamespaceReplayer: 0x5625a2f0c700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:24.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.560+0000 7f2403250640 -1 rbd::mirror::MirrorStatusUpdater 0x5625a3fe6120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:24.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.560+0000 7f2409a5d640 -1 rbd::mirror::NamespaceReplayer: 0x5625a2f0ca80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:24.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.560+0000 7f2403a51640 -1 rbd::mirror::MirrorStatusUpdater 0x5625a3fe6240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:24.563 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.560+0000 7f2409a5d640 -1 rbd::mirror::NamespaceReplayer: 0x5625a2f0ce00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:24.610 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.607+0000 7f23f6236640 -1 rbd::mirror::InstanceWatcher: 0x5625a449a380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:24.610 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.607+0000 7f23f5a35640 -1 rbd::mirror::InstanceWatcher: 0x5625a449ae00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:24.611 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:24.609+0000 7f2409a5d640 -1 rbd::mirror::LeaderWatcher: 0x5625a4d02300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:25.056 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.054+0000 7fb96f623640 -1 rbd::mirror::RemotePoolPoller: 0x559f009f6dd0 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2026-02-20T21:46:25.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.054+0000 7fb96ee22640 -1 rbd::mirror::RemotePoolPoller: 0x559f009f6dd0 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2026-02-20T21:46:25.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.054+0000 7fb96ee22640 -1 rbd::mirror::PoolReplayer: 0x559f01796000 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2026-02-20T21:46:25.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.054+0000 7fb975e30640 -1 rbd::mirror::InstanceWatcher: 0x559f01b3c000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.054+0000 7fb97562f640 -1 rbd::mirror::InstanceWatcher: 0x559f01b3ca80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.054+0000 7fb97562f640 -1 rbd::mirror::InstanceWatcher: 0x559f01b3c380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.055+0000 7fb96f623640 -1 rbd::mirror::MirrorStatusUpdater 0x559f01aba480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.055+0000 7fb97c63d640 -1 rbd::mirror::NamespaceReplayer: 0x559f00a0afc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.055+0000 7fb96ee22640 -1 rbd::mirror::MirrorStatusUpdater 0x559f01aba240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.055+0000 7fb97be3c640 -1 rbd::mirror::NamespaceReplayer: 0x559f024de000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.057 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.055+0000 7fb96f623640 -1 rbd::mirror::MirrorStatusUpdater 0x559f01aba360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.058 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.055+0000 7fb97c63d640 -1 rbd::mirror::NamespaceReplayer: 0x559f00a0ae00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.080 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.079+0000 7fb961e08640 -1 rbd::mirror::RemotePoolPoller: 0x559f009f6820 handle_mirror_peer_ping: failed to ping remote mirror peer: (2) No such file or directory 2026-02-20T21:46:25.080 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.079+0000 7fb961607640 -1 rbd::mirror::RemotePoolPoller: 0x559f009f6820 handle_mirror_peer_list: failed to retrieve mirror peers: (2) No such file or directory 2026-02-20T21:46:25.080 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.079+0000 7fb961607640 -1 rbd::mirror::PoolReplayer: 0x559f027dc600 handle_remote_pool_meta_updated: remote pool metadata updated unexpectedly 2026-02-20T21:46:25.083 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.080+0000 7fb961e08640 -1 rbd::mirror::MirrorStatusUpdater 0x559f01d44480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.083 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.080+0000 7fb97c63d640 -1 rbd::mirror::NamespaceReplayer: 0x559f024df340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.084 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.082+0000 7fb961607640 -1 rbd::mirror::MirrorStatusUpdater 0x559f01aba7e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.084 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.082+0000 7fb97be3c640 -1 rbd::mirror::NamespaceReplayer: 0x559f024df180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.087 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.085+0000 7fb97c63d640 -1 rbd::mirror::LeaderWatcher: 0x559f027dc300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:25.089 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.086+0000 7fb97fe9ea40 -1 rbd::mirror::Mirror: 0x559f00a66240 update_pool_replayers: restarting failed pool replayer for uuid: 5e679c20-44a3-436b-85ec-ee9ecf7ffe81 cluster: cluster1 client: client.mirror 2026-02-20T21:46:25.111 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.108+0000 7fb97fe9ea40 -1 rbd::mirror::PoolReplayer: 0x559f027dc600 init: error accessing local pool 4: (2) No such file or directory 2026-02-20T21:46:25.164 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:pool 'mirror_parent' does not exist 2026-02-20T21:46:25.172 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-20T21:46:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirrors cluster1 2026-02-20T21:46:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1 2026-02-20T21:46:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-20T21:46:25.173 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:46:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster1:0 '' 2026-02-20T21:46:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:0 2026-02-20T21:46:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-20T21:46:25.177 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:46:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:46:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster1:1 '' 2026-02-20T21:46:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:1 2026-02-20T21:46:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-20T21:46:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:46:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:46:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster1:2 '' 2026-02-20T21:46:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:2 2026-02-20T21:46:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-20T21:46:25.178 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:46:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:46:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster1:3 '' 2026-02-20T21:46:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster1:3 2026-02-20T21:46:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-20T21:46:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:46:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:46:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-20T21:46:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirrors cluster2 2026-02-20T21:46:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2 2026-02-20T21:46:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-20T21:46:25.179 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:++ seq 0 3 2026-02-20T21:46:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster2:0 '' 2026-02-20T21:46:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:0 2026-02-20T21:46:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-20T21:46:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:46:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:46:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster2:1 '' 2026-02-20T21:46:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:1 2026-02-20T21:46:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-20T21:46:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:46:25.180 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:46:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster2:2 '' 2026-02-20T21:46:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:2 2026-02-20T21:46:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-20T21:46:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:46:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:46:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-20T21:46:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ stop_mirror cluster2:3 '' 2026-02-20T21:46:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local cluster=cluster2:3 2026-02-20T21:46:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ local sig= 2026-02-20T21:46:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test -n 1 2026-02-20T21:46:25.181 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ return 2026-02-20T21:46:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' -z 1 ']' 2026-02-20T21:46:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ test '' = /tmp/tmp.IQNOxgiLiE 2026-02-20T21:46:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ rm -Rf /tmp/tmp.IQNOxgiLiE 2026-02-20T21:46:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ '[' 0 -eq 0 ']' 2026-02-20T21:46:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ echo OK 2026-02-20T21:46:25.182 INFO:tasks.workunit.cluster1.client.mirror.trial163.stdout:OK 2026-02-20T21:46:25.183 INFO:tasks.workunit.cluster1.client.mirror.trial163.stderr:+ exit 0 2026-02-20T21:46:25.184 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-20T21:46:25.184 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-20T21:46:25.252 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_stress.sh'] on cluster1.client.mirror... 2026-02-20T21:46:25.252 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-20T21:46:25.484 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.482+0000 7faa4d02b640 -1 rbd::mirror::LeaderWatcher: 0x561ebfd17b00 handle_notify_heartbeat: error notifying heartbeat: (2) No such file or directory, releasing leader 2026-02-20T21:46:25.485 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.482+0000 7faa327f6640 -1 rbd::mirror::PoolWatcher: 0x561ebe8fad20 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:25.486 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.482+0000 7faa327f6640 -1 rbd::mirror::PoolWatcher: 0x561ec29aa1e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:25.486 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.482+0000 7faa4d02b640 -1 rbd::mirror::LeaderWatcher: 0x561ebfd17b00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-20T21:46:25.486 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.482+0000 7faa4d02b640 -1 rbd::mirror::LeaderWatcher: 0x561ebfd17b00 handle_release_leader_lock: error releasing lock: (2) No such file or directory 2026-02-20T21:46:25.559 DEBUG:teuthology.parallel:result is None 2026-02-20T21:46:25.559 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-20T21:46:25.583 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-20T21:46:25.583 DEBUG:teuthology.orchestra.run.trial163:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-20T21:46:25.640 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-20T21:46:25.640 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-02-20T21:46:25.660 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-02-20T21:46:25.660 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 18 2026-02-20T21:46:25.661 INFO:tasks.rbd_mirror_thrash:done joining 2026-02-20T21:46:25.661 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror-thrash 2026-02-20T21:46:25.672 INFO:tasks.rbd_mirror_thrash:joining rbd_mirror_thrash 2026-02-20T21:46:25.672 INFO:tasks.rbd_mirror_thrash:stat['kill'] = 17 2026-02-20T21:46:25.672 INFO:tasks.rbd_mirror_thrash:done joining 2026-02-20T21:46:25.672 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T21:46:25.686 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.3:waiting for process to exit 2026-02-20T21:46:25.687 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:25.687 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.685+0000 7f8f7843c640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.3 (PID: 114635) UID: 1000 2026-02-20T21:46:25.687 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.685+0000 7f8f77c3b640 -1 rbd::mirror::InstanceWatcher: 0x562e90499180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.10359 object: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.685+0000 7f8f77c3b640 -1 rbd::mirror::InstanceWatcher: 0x562e90499500 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.10359 object: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.685+0000 7f8f77c3b640 -1 rbd::mirror::InstanceWatcher: 0x562e9032c380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.10359 object: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.685+0000 7f8f77c3b640 -1 rbd::mirror::InstanceWatcher: 0x562e8fe17c00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.10362 object: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.685+0000 7f8f77c3b640 -1 rbd::mirror::InstanceWatcher: 0x562e9032c000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.10362 object: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.685+0000 7f8f7142e640 -1 rbd::mirror::InstanceWatcher: 0x562e90499180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.685+0000 7f8f70c2d640 -1 rbd::mirror::InstanceWatcher: 0x562e90499500 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.685+0000 7f8f63c13640 -1 rbd::mirror::InstanceWatcher: 0x562e8fe17c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.685+0000 7f8f7142e640 -1 rbd::mirror::InstanceWatcher: 0x562e9032c380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.685+0000 7f8f63412640 -1 rbd::mirror::InstanceWatcher: 0x562e9032c000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f6ac21640 -1 rbd::mirror::MirrorStatusUpdater 0x562e8fd48360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f6ac21640 -1 rbd::mirror::MirrorStatusUpdater 0x562e8fd485a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f7743a640 -1 rbd::mirror::NamespaceReplayer: 0x562e8e972a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f7743a640 -1 rbd::mirror::NamespaceReplayer: 0x562e8e972e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f5d406640 -1 rbd::mirror::MirrorStatusUpdater 0x562e8ffbad80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f6a420640 -1 rbd::mirror::MirrorStatusUpdater 0x562e8fd48480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f77c3b640 -1 rbd::mirror::NamespaceReplayer: 0x562e8fd4afc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.688 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f77c3b640 -1 rbd::mirror::NamespaceReplayer: 0x562e8fd4a000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.689 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f5cc05640 -1 rbd::mirror::MirrorStatusUpdater 0x562e8f89fb00 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.689 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f77c3b640 -1 rbd::mirror::NamespaceReplayer: 0x562e8fd4b180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.689 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f7142e640 -1 rbd::mirror::MirrorStatusUpdater 0x562e8fd48000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.689 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f7142e640 -1 rbd::mirror::MirrorStatusUpdater 0x562e8fd48120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.689 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f77c3b640 -1 rbd::mirror::NamespaceReplayer: 0x562e8e972a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.689 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f77c3b640 -1 rbd::mirror::NamespaceReplayer: 0x562e8e972e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.689 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f63c13640 -1 rbd::mirror::MirrorStatusUpdater 0x562e8ffbaa20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.689 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f63c13640 -1 rbd::mirror::MirrorStatusUpdater 0x562e8ffbab40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.689 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f7743a640 -1 rbd::mirror::NamespaceReplayer: 0x562e8fd4afc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.689 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f7743a640 -1 rbd::mirror::NamespaceReplayer: 0x562e8fd4b180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.689 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.686+0000 7f8f77c3b640 -1 rbd::mirror::LeaderWatcher: 0x562e90023b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:25.690 INFO:tasks.rbd_mirror.cluster2.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.688+0000 7f8f77c3b640 -1 rbd::mirror::LeaderWatcher: 0x562e8fb1a300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:25.700 INFO:tasks.rbd_mirror.cluster2.client.mirror.3:Stopped 2026-02-20T21:46:25.700 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T21:46:25.711 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.2:waiting for process to exit 2026-02-20T21:46:25.711 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:25.741 INFO:tasks.rbd_mirror.cluster2.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.739+0000 7fb97ce3e640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.2 (PID: 141661) UID: 1000 2026-02-20T21:46:25.750 INFO:tasks.rbd_mirror.cluster2.client.mirror.2:Stopped 2026-02-20T21:46:25.750 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T21:46:25.762 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.1:waiting for process to exit 2026-02-20T21:46:25.762 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:25.791 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.789+0000 7f0e53dcf640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.1 (PID: 140526) UID: 1000 2026-02-20T21:46:25.791 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.789+0000 7f0e52dcd640 -1 rbd::mirror::InstanceWatcher: 0x56247b19dc00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.12495 object: (2) No such file or directory 2026-02-20T21:46:25.791 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.789+0000 7f0e52dcd640 -1 rbd::mirror::InstanceWatcher: 0x56247a70a700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.12495 object: (2) No such file or directory 2026-02-20T21:46:25.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.789+0000 7f0e3eda5640 -1 rbd::mirror::InstanceWatcher: 0x56247b19dc00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.789+0000 7f0e3eda5640 -1 rbd::mirror::InstanceWatcher: 0x56247a70a700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.790+0000 7f0e38598640 -1 rbd::mirror::MirrorStatusUpdater 0x56247a7530e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.790+0000 7f0e38598640 -1 rbd::mirror::MirrorStatusUpdater 0x56247a752d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.790+0000 7f0e52dcd640 -1 rbd::mirror::NamespaceReplayer: 0x56247a4dcfc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.790+0000 7f0e52dcd640 -1 rbd::mirror::NamespaceReplayer: 0x56247a4dd180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.790+0000 7f0e3f5a6640 -1 rbd::mirror::MirrorStatusUpdater 0x56247a752a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.790+0000 7f0e3f5a6640 -1 rbd::mirror::MirrorStatusUpdater 0x56247a752b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.790+0000 7f0e535ce640 -1 rbd::mirror::NamespaceReplayer: 0x56247a4dcfc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.790+0000 7f0e52dcd640 -1 rbd::mirror::NamespaceReplayer: 0x56247a4dd180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.792 INFO:tasks.rbd_mirror.cluster2.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.790+0000 7f0e52dcd640 -1 rbd::mirror::LeaderWatcher: 0x56247a7b5b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:25.803 INFO:tasks.rbd_mirror.cluster2.client.mirror.1:Stopped 2026-02-20T21:46:25.803 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T21:46:25.822 DEBUG:tasks.rbd_mirror.cluster2.client.mirror.0:waiting for process to exit 2026-02-20T21:46:25.822 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:25.847 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe95214640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster2 --id mirror.0 (PID: 135830) UID: 1000 2026-02-20T21:46:25.847 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe801ea640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x556fd6201d40 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T21:46:25.848 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe801ea640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x556fd6201b00 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T21:46:25.848 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe809eb640 -1 rbd::mirror::PoolWatcher: 0x556fd65f8f00 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:25.848 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe809eb640 -1 rbd::mirror::PoolWatcher: 0x556fd65f8000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:25.848 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe94a13640 -1 rbd::mirror::InstanceWatcher: 0x556fd6447180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.12302 object: (2) No such file or directory 2026-02-20T21:46:25.848 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe94a13640 -1 rbd::mirror::InstanceWatcher: 0x556fd593c380 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.12302 object: (2) No such file or directory 2026-02-20T21:46:25.848 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe94a13640 -1 rbd::mirror::InstanceWatcher: 0x556fd593c000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.12302 object: (2) No such file or directory 2026-02-20T21:46:25.848 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe94212640 -1 rbd::mirror::InstanceWatcher: 0x556fd5faa700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.12306 object: (2) No such file or directory 2026-02-20T21:46:25.848 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe8da05640 -1 rbd::mirror::InstanceWatcher: 0x556fd6447180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.848 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe8e206640 -1 rbd::mirror::InstanceWatcher: 0x556fd593c380 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.848 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe8da05640 -1 rbd::mirror::InstanceWatcher: 0x556fd593c000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.842+0000 7efe94212640 -1 rbd::mirror::InstanceWatcher: 0x556fd593c700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.12306 object: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe801ea640 -1 rbd::mirror::InstanceWatcher: 0x556fd5faa700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe879f9640 -1 rbd::mirror::MirrorStatusUpdater 0x556fd5d5a360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe879f9640 -1 rbd::mirror::MirrorStatusUpdater 0x556fd5d5a480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe94212640 -1 rbd::mirror::NamespaceReplayer: 0x556fd5d58000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe871f8640 -1 rbd::mirror::MirrorStatusUpdater 0x556fd5d5a5a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe809eb640 -1 rbd::mirror::InstanceWatcher: 0x556fd593c700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe94212640 -1 rbd::mirror::NamespaceReplayer: 0x556fd5d58380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe94212640 -1 rbd::mirror::NamespaceReplayer: 0x556fd5d581c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe801ea640 -1 rbd::mirror::MirrorStatusUpdater 0x556fd5fd6a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe94212640 -1 rbd::mirror::NamespaceReplayer: 0x556fd5d58fc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe8e206640 -1 rbd::mirror::MirrorStatusUpdater 0x556fd5d5a000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.849 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe94a13640 -1 rbd::mirror::NamespaceReplayer: 0x556fd5d58000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.850 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe8da05640 -1 rbd::mirror::MirrorStatusUpdater 0x556fd5d5a120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.850 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe94a13640 -1 rbd::mirror::NamespaceReplayer: 0x556fd5d581c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.850 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe809eb640 -1 rbd::mirror::MirrorStatusUpdater 0x556fd5fd6b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.850 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe94212640 -1 rbd::mirror::NamespaceReplayer: 0x556fd5d59180 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.850 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe94212640 -1 rbd::mirror::LeaderWatcher: 0x556fd6047b00 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-20T21:46:25.850 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe94212640 -1 rbd::mirror::LeaderWatcher: 0x556fd6047b00 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-20T21:46:25.850 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.843+0000 7efe94a13640 -1 rbd::mirror::LeaderWatcher: 0x556fd6047b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:25.850 INFO:tasks.rbd_mirror.cluster2.client.mirror.0.trial163.stderr:2026-02-20T21:46:25.845+0000 7efe94a13640 -1 rbd::mirror::LeaderWatcher: 0x556fd59f8300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:25.857 INFO:tasks.rbd_mirror.cluster2.client.mirror.0:Stopped 2026-02-20T21:46:25.857 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T21:46:25.875 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.3:waiting for process to exit 2026-02-20T21:46:25.876 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:25.897 INFO:tasks.rbd_mirror.cluster1.client.mirror.3.trial163.stderr:2026-02-20T21:46:25.895+0000 7f240aa5f640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.3 (PID: 142354) UID: 1000 2026-02-20T21:46:25.905 INFO:tasks.rbd_mirror.cluster1.client.mirror.3:Stopped 2026-02-20T21:46:25.906 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T21:46:25.925 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.2:waiting for process to exit 2026-02-20T21:46:25.925 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:25.946 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.944+0000 7f4a8ebbe640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.2 (PID: 138933) UID: 1000 2026-02-20T21:46:25.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.944+0000 7f4a8dbbc640 -1 rbd::mirror::InstanceWatcher: 0x556658ed9c00 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5323 object: (2) No such file or directory 2026-02-20T21:46:25.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.944+0000 7f4a8e3bd640 -1 rbd::mirror::InstanceWatcher: 0x55665847a700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5323 object: (2) No such file or directory 2026-02-20T21:46:25.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.944+0000 7f4a8dbbc640 -1 rbd::mirror::InstanceWatcher: 0x556658bff180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5320 object: (2) No such file or directory 2026-02-20T21:46:25.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.944+0000 7f4a8e3bd640 -1 rbd::mirror::InstanceWatcher: 0x556658ed9880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5320 object: (2) No such file or directory 2026-02-20T21:46:25.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.944+0000 7f4a8dbbc640 -1 rbd::mirror::InstanceWatcher: 0x556658ed8000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5320 object: (2) No such file or directory 2026-02-20T21:46:25.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a79b94640 -1 rbd::mirror::InstanceWatcher: 0x556658ed9c00 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a7a395640 -1 rbd::mirror::InstanceWatcher: 0x55665847a700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.950 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a87bb0640 -1 rbd::mirror::InstanceWatcher: 0x556658bff180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a73b88640 -1 rbd::mirror::MirrorStatusUpdater 0x556658228c60 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a73b88640 -1 rbd::mirror::MirrorStatusUpdater 0x556658228900 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a8e3bd640 -1 rbd::mirror::NamespaceReplayer: 0x556658227340 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a8e3bd640 -1 rbd::mirror::NamespaceReplayer: 0x556658227500 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a87bb0640 -1 rbd::mirror::InstanceWatcher: 0x556658ed9880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a87bb0640 -1 rbd::mirror::InstanceWatcher: 0x556658ed8000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:25.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a79b94640 -1 rbd::mirror::MirrorStatusUpdater 0x5566584a8a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a80ba2640 -1 rbd::mirror::MirrorStatusUpdater 0x556658228360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a7a395640 -1 rbd::mirror::MirrorStatusUpdater 0x5566584a8b40 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a8dbbc640 -1 rbd::mirror::NamespaceReplayer: 0x556658227340 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a8dbbc640 -1 rbd::mirror::NamespaceReplayer: 0x556658226000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.951 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a8dbbc640 -1 rbd::mirror::NamespaceReplayer: 0x556658227500 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a813a3640 -1 rbd::mirror::MirrorStatusUpdater 0x556658228480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a813a3640 -1 rbd::mirror::MirrorStatusUpdater 0x5566582285a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a8dbbc640 -1 rbd::mirror::NamespaceReplayer: 0x556658226380 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a8dbbc640 -1 rbd::mirror::NamespaceReplayer: 0x5566582261c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a873af640 -1 rbd::mirror::MirrorStatusUpdater 0x556658228000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a8e3bd640 -1 rbd::mirror::NamespaceReplayer: 0x556658226000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a87bb0640 -1 rbd::mirror::MirrorStatusUpdater 0x556658228240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a8e3bd640 -1 rbd::mirror::NamespaceReplayer: 0x556658226380 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a8e3bd640 -1 rbd::mirror::LeaderWatcher: 0x55665852bb00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:25.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a873af640 -1 rbd::mirror::MirrorStatusUpdater 0x556658228120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:25.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.945+0000 7f4a8e3bd640 -1 rbd::mirror::NamespaceReplayer: 0x5566582261c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:25.952 INFO:tasks.rbd_mirror.cluster1.client.mirror.2.trial163.stderr:2026-02-20T21:46:25.947+0000 7f4a8e3bd640 -1 rbd::mirror::LeaderWatcher: 0x556658f3e300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:25.959 INFO:tasks.rbd_mirror.cluster1.client.mirror.2:Stopped 2026-02-20T21:46:25.960 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T21:46:25.980 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.1:waiting for process to exit 2026-02-20T21:46:25.981 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:26.000 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.998+0000 7faa4e02d640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.1 (PID: 90608) UID: 1000 2026-02-20T21:46:26.004 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.998+0000 7faa4701f640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x561ec00778c0 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T21:46:26.004 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.998+0000 7faa4701f640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x561ec5044480 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T21:46:26.004 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.998+0000 7faa4701f640 -1 rbd::mirror::image_deleter::TrashWatcher: 0x561ec5044240 handle_unregister_watcher: error unregistering watcher for trash directory: (2) No such file or directory 2026-02-20T21:46:26.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.998+0000 7faa4681e640 -1 rbd::mirror::PoolWatcher: 0x561ec62345a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:26.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.998+0000 7faa4701f640 -1 rbd::mirror::PoolWatcher: 0x561ec51e34a0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:26.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.998+0000 7faa40011640 -1 rbd::mirror::PoolWatcher: 0x561ec51e2780 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:26.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.998+0000 7faa40011640 -1 rbd::mirror::PoolWatcher: 0x561ec51e3680 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:26.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.998+0000 7faa40011640 -1 rbd::mirror::PoolWatcher: 0x561ebff96000 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:26.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.998+0000 7faa4d82c640 -1 rbd::mirror::InstanceWatcher: 0x561ec0558700 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4629 object: (2) No such file or directory 2026-02-20T21:46:26.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.998+0000 7faa4681e640 -1 rbd::mirror::PoolWatcher: 0x561ec51e30e0 operator(): error unregistering watcher for rbd_mirroring object: (2) No such file or directory 2026-02-20T21:46:26.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa39804640 -1 rbd::mirror::InstanceWatcher: 0x561ec003c000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:26.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa39003640 -1 rbd::mirror::InstanceWatcher: 0x561ec0558700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:26.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa327f6640 -1 rbd::mirror::MirrorStatusUpdater 0x561ebfcb4d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.005 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa327f6640 -1 rbd::mirror::MirrorStatusUpdater 0x561ebfcb5320 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d02b640 -1 rbd::mirror::NamespaceReplayer: 0x561ebfa2f180 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d02b640 -1 rbd::mirror::NamespaceReplayer: 0x561ebfa2efc0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d02b640 -1 rbd::mirror::InstanceWatcher: 0x561ec0558000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4626 object: (2) No such file or directory 2026-02-20T21:46:26.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d02b640 -1 rbd::mirror::InstanceWatcher: 0x561ec0558a80 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4626 object: (2) No such file or directory 2026-02-20T21:46:26.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa39804640 -1 rbd::mirror::MirrorStatusUpdater 0x561ebfcb4a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d82c640 -1 rbd::mirror::NamespaceReplayer: 0x561ebfa2efc0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d02b640 -1 rbd::mirror::InstanceWatcher: 0x561ec0407180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.4626 object: (2) No such file or directory 2026-02-20T21:46:26.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d02b640 -1 rbd::mirror::LeaderWatcher: 0x561ebfd17b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:26.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4681e640 -1 rbd::mirror::InstanceWatcher: 0x561ec0558000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:26.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4701f640 -1 rbd::mirror::InstanceWatcher: 0x561ec0558a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:26.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4681e640 -1 rbd::mirror::InstanceWatcher: 0x561ec0407180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:26.006 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa40812640 -1 rbd::mirror::MirrorStatusUpdater 0x561ebf9d6a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa40812640 -1 rbd::mirror::MirrorStatusUpdater 0x561ebf9d66c0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d82c640 -1 rbd::mirror::NamespaceReplayer: 0x561ebfa2e000 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa40011640 -1 rbd::mirror::MirrorStatusUpdater 0x561ebf9d65a0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d82c640 -1 rbd::mirror::NamespaceReplayer: 0x561ebfa2e1c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d82c640 -1 rbd::mirror::NamespaceReplayer: 0x561ebe954e00 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4701f640 -1 rbd::mirror::MirrorStatusUpdater 0x561ebf9d6360 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d02b640 -1 rbd::mirror::NamespaceReplayer: 0x561ebfa2e000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4701f640 -1 rbd::mirror::MirrorStatusUpdater 0x561ebf9d6480 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4701f640 -1 rbd::mirror::MirrorStatusUpdater 0x561ebf9d6240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d02b640 -1 rbd::mirror::NamespaceReplayer: 0x561ebfa2e1c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:25.999+0000 7faa4d82c640 -1 rbd::mirror::NamespaceReplayer: 0x561ebe954e00 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.007 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:26.001+0000 7faa4d02b640 -1 rbd::mirror::LeaderWatcher: 0x561ec0752300 handle_notify_lock_released: error notifying leader lock released: (2) No such file or directory 2026-02-20T21:46:26.008 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:26.001+0000 7faa4d02b640 -1 rbd::mirror::LeaderWatcher: 0x561ec0752300 handle_shut_down_leader_lock: error shutting down leader lock: (2) No such file or directory 2026-02-20T21:46:26.008 INFO:tasks.rbd_mirror.cluster1.client.mirror.1.trial163.stderr:2026-02-20T21:46:26.001+0000 7faa4d82c640 -1 rbd::mirror::LeaderWatcher: 0x561ec0752300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:26.018 INFO:tasks.rbd_mirror.cluster1.client.mirror.1:Stopped 2026-02-20T21:46:26.018 DEBUG:teuthology.run_tasks:Unwinding manager rbd-mirror 2026-02-20T21:46:26.037 DEBUG:tasks.rbd_mirror.cluster1.client.mirror.0:waiting for process to exit 2026-02-20T21:46:26.037 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:26.059 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.057+0000 7f19b5386640 -1 received signal: Terminated from /usr/bin/python3 /usr/bin/daemon-helper term rbd-mirror --foreground --cluster cluster1 --id mirror.0 (PID: 141234) UID: 1000 2026-02-20T21:46:26.062 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.057+0000 7f19b4384640 -1 rbd::mirror::InstanceWatcher: 0x560633ddf180 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5430 object: (2) No such file or directory 2026-02-20T21:46:26.062 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.057+0000 7f19b4384640 -1 rbd::mirror::InstanceWatcher: 0x560633f2f880 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5430 object: (2) No such file or directory 2026-02-20T21:46:26.062 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.057+0000 7f19b4384640 -1 rbd::mirror::InstanceWatcher: 0x560633f2e000 handle_unregister_watch: error unregistering instance watcher for rbd_mirror_instance.5430 object: (2) No such file or directory 2026-02-20T21:46:26.062 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.057+0000 7f19a0b5d640 -1 rbd::mirror::InstanceWatcher: 0x560633668700 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:26.062 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.057+0000 7f19a035c640 -1 rbd::mirror::InstanceWatcher: 0x560633668a80 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:26.062 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19adb77640 -1 rbd::mirror::InstanceWatcher: 0x560633ddf180 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:26.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19ae378640 -1 rbd::mirror::InstanceWatcher: 0x560633f2f880 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:26.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19adb77640 -1 rbd::mirror::InstanceWatcher: 0x560633f2e000 handle_unregister_instance: error unregistering instance: (2) No such file or directory 2026-02-20T21:46:26.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f199a350640 -1 rbd::mirror::MirrorStatusUpdater 0x5606336950e0 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19ae378640 -1 rbd::mirror::MirrorStatusUpdater 0x560633410000 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f199a350640 -1 rbd::mirror::MirrorStatusUpdater 0x560633694d80 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19b4384640 -1 rbd::mirror::NamespaceReplayer: 0x5606335a88c0 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19b4384640 -1 rbd::mirror::NamespaceReplayer: 0x560632324000 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19b4384640 -1 rbd::mirror::NamespaceReplayer: 0x5606335a8a80 handle_shut_down_remote_status_updater: error shutting remote mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19ae378640 -1 rbd::mirror::MirrorStatusUpdater 0x560633410240 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19ae378640 -1 rbd::mirror::MirrorStatusUpdater 0x560633410120 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19b4b85640 -1 rbd::mirror::NamespaceReplayer: 0x560632324a80 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.063 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19b4b85640 -1 rbd::mirror::NamespaceReplayer: 0x560632324700 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.064 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19a0b5d640 -1 rbd::mirror::MirrorStatusUpdater 0x560633694a20 handle_shut_down_mirror_status_watcher: failed to shut down mirror status watcher: (2) No such file or directory 2026-02-20T21:46:26.064 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19b4384640 -1 rbd::mirror::NamespaceReplayer: 0x5606335a88c0 handle_shut_down_local_status_updater: error shutting local mirror status updater down: (2) No such file or directory 2026-02-20T21:46:26.064 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.058+0000 7f19b4384640 -1 rbd::mirror::LeaderWatcher: 0x560633717b00 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:26.064 INFO:tasks.rbd_mirror.cluster1.client.mirror.0.trial163.stderr:2026-02-20T21:46:26.060+0000 7f19b4384640 -1 rbd::mirror::LeaderWatcher: 0x560634106300 handle_unregister_watch: error unregistering leader watcher for rbd_mirror_leader object: (2) No such file or directory 2026-02-20T21:46:26.071 INFO:tasks.rbd_mirror.cluster1.client.mirror.0:Stopped 2026-02-20T21:46:26.071 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-20T21:46:26.091 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-20T21:46:26.092 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:46:26.293 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:46:26.293 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-20T21:46:26.301 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":1641,"stamp":"2026-02-20T21:46:25.097161+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":5181084,"kb_used_data":619692,"kb_used_omap":559,"kb_used_meta":4560784,"kb_avail":277934436,"statfs":{"total":289910292480,"available":284604862464,"internally_reserved":0,"allocated":634564608,"data_stored":625656923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":572601,"internal_metadata":4670243655},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-1383614684,"num_objects":-226,"num_object_clones":0,"num_object_copies":-452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-226,"num_whiteouts":0,"num_read":-2398110,"num_read_kb":-89552721,"num_write":-2345210,"num_write_kb":-91560408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.120587"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":828,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.092444+0000","last_change":"2026-02-20T21:05:32.885706+0000","last_active":"2026-02-20T21:46:20.092444+0000","last_peered":"2026-02-20T21:46:20.092444+0000","last_clean":"2026-02-20T21:46:20.092444+0000","last_became_active":"2026-02-20T21:05:30.883104+0000","last_became_peered":"2026-02-20T21:05:30.883104+0000","last_unstale":"2026-02-20T21:46:20.092444+0000","last_undegraded":"2026-02-20T21:46:20.092444+0000","last_fullsized":"2026-02-20T21:46:20.092444+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:14:32.329895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010312629999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":828,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.092569+0000","last_change":"2026-02-20T21:05:32.885327+0000","last_active":"2026-02-20T21:46:20.092569+0000","last_peered":"2026-02-20T21:46:20.092569+0000","last_clean":"2026-02-20T21:46:20.092569+0000","last_became_active":"2026-02-20T21:05:30.882952+0000","last_became_peered":"2026-02-20T21:05:30.882952+0000","last_unstale":"2026-02-20T21:46:20.092569+0000","last_undegraded":"2026-02-20T21:46:20.092569+0000","last_fullsized":"2026-02-20T21:46:20.092569+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:32:38.163582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071972500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":829,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091185+0000","last_change":"2026-02-20T21:05:32.885072+0000","last_active":"2026-02-20T21:46:20.091185+0000","last_peered":"2026-02-20T21:46:20.091185+0000","last_clean":"2026-02-20T21:46:20.091185+0000","last_became_active":"2026-02-20T21:05:30.883044+0000","last_became_peered":"2026-02-20T21:05:30.883044+0000","last_unstale":"2026-02-20T21:46:20.091185+0000","last_undegraded":"2026-02-20T21:46:20.091185+0000","last_fullsized":"2026-02-20T21:46:20.091185+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:57:08.905691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055306500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":829,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.090983+0000","last_change":"2026-02-20T21:05:32.885039+0000","last_active":"2026-02-20T21:46:20.090983+0000","last_peered":"2026-02-20T21:46:20.090983+0000","last_clean":"2026-02-20T21:46:20.090983+0000","last_became_active":"2026-02-20T21:05:30.883115+0000","last_became_peered":"2026-02-20T21:05:30.883115+0000","last_unstale":"2026-02-20T21:46:20.090983+0000","last_undegraded":"2026-02-20T21:46:20.090983+0000","last_fullsized":"2026-02-20T21:46:20.090983+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:11:05.945358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052496400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":831,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091395+0000","last_change":"2026-02-20T21:05:32.883927+0000","last_active":"2026-02-20T21:46:20.091395+0000","last_peered":"2026-02-20T21:46:20.091395+0000","last_clean":"2026-02-20T21:46:20.091395+0000","last_became_active":"2026-02-20T21:05:30.883032+0000","last_became_peered":"2026-02-20T21:05:30.883032+0000","last_unstale":"2026-02-20T21:46:20.091395+0000","last_undegraded":"2026-02-20T21:46:20.091395+0000","last_fullsized":"2026-02-20T21:46:20.091395+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:07:25.195375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077015799999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":827,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091171+0000","last_change":"2026-02-20T21:05:32.927062+0000","last_active":"2026-02-20T21:46:20.091171+0000","last_peered":"2026-02-20T21:46:20.091171+0000","last_clean":"2026-02-20T21:46:20.091171+0000","last_became_active":"2026-02-20T21:05:30.883679+0000","last_became_peered":"2026-02-20T21:05:30.883679+0000","last_unstale":"2026-02-20T21:46:20.091171+0000","last_undegraded":"2026-02-20T21:46:20.091171+0000","last_fullsized":"2026-02-20T21:46:20.091171+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:43:48.880973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021062000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":828,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091655+0000","last_change":"2026-02-20T21:05:32.927079+0000","last_active":"2026-02-20T21:46:20.091655+0000","last_peered":"2026-02-20T21:46:20.091655+0000","last_clean":"2026-02-20T21:46:20.091655+0000","last_became_active":"2026-02-20T21:05:30.883572+0000","last_became_peered":"2026-02-20T21:05:30.883572+0000","last_unstale":"2026-02-20T21:46:20.091655+0000","last_undegraded":"2026-02-20T21:46:20.091655+0000","last_fullsized":"2026-02-20T21:46:20.091655+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:29:24.098186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022568800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":830,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.092509+0000","last_change":"2026-02-20T21:05:32.885282+0000","last_active":"2026-02-20T21:46:20.092509+0000","last_peered":"2026-02-20T21:46:20.092509+0000","last_clean":"2026-02-20T21:46:20.092509+0000","last_became_active":"2026-02-20T21:05:30.882188+0000","last_became_peered":"2026-02-20T21:05:30.882188+0000","last_unstale":"2026-02-20T21:46:20.092509+0000","last_undegraded":"2026-02-20T21:46:20.092509+0000","last_fullsized":"2026-02-20T21:46:20.092509+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:59:42.030633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075006200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":921,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091293+0000","last_change":"2026-02-20T21:05:28.168843+0000","last_active":"2026-02-20T21:46:20.091293+0000","last_peered":"2026-02-20T21:46:20.091293+0000","last_clean":"2026-02-20T21:46:20.091293+0000","last_became_active":"2026-02-20T21:05:28.168224+0000","last_became_peered":"2026-02-20T21:05:28.168224+0000","last_unstale":"2026-02-20T21:46:20.091293+0000","last_undegraded":"2026-02-20T21:46:20.091293+0000","last_fullsized":"2026-02-20T21:46:20.091293+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:26.870888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:26.870888+0000","last_clean_scrub_stamp":"2026-02-20T21:05:26.870888+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:36:01.610328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738862,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1404332,"kb_used_data":4764,"kb_used_omap":94,"kb_used_meta":1399457,"kb_avail":92967508,"statfs":{"total":96636764160,"available":95198728192,"internally_reserved":0,"allocated":4878336,"data_stored":2255193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":96524,"internal_metadata":1433044724},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738863,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2162564,"kb_used_data":276916,"kb_used_omap":313,"kb_used_meta":1885318,"kb_avail":92209276,"statfs":{"total":96636764160,"available":94422298624,"internally_reserved":0,"allocated":283561984,"data_stored":280452985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":321013,"internal_metadata":1930566155},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738863,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1614188,"kb_used_data":338012,"kb_used_omap":151,"kb_used_meta":1276008,"kb_avail":92757652,"statfs":{"total":96636764160,"available":94983835648,"internally_reserved":0,"allocated":346124288,"data_stored":342948745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":155064,"internal_metadata":1306632776},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:46:26.303 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:46:26.485 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:46:26.485 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-20T21:46:26.494 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":1641,"stamp":"2026-02-20T21:46:25.097161+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":5181084,"kb_used_data":619692,"kb_used_omap":559,"kb_used_meta":4560784,"kb_avail":277934436,"statfs":{"total":289910292480,"available":284604862464,"internally_reserved":0,"allocated":634564608,"data_stored":625656923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":572601,"internal_metadata":4670243655},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-1383614684,"num_objects":-226,"num_object_clones":0,"num_object_copies":-452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-226,"num_whiteouts":0,"num_read":-2398110,"num_read_kb":-89552721,"num_write":-2345210,"num_write_kb":-91560408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.120587"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":828,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.092444+0000","last_change":"2026-02-20T21:05:32.885706+0000","last_active":"2026-02-20T21:46:20.092444+0000","last_peered":"2026-02-20T21:46:20.092444+0000","last_clean":"2026-02-20T21:46:20.092444+0000","last_became_active":"2026-02-20T21:05:30.883104+0000","last_became_peered":"2026-02-20T21:05:30.883104+0000","last_unstale":"2026-02-20T21:46:20.092444+0000","last_undegraded":"2026-02-20T21:46:20.092444+0000","last_fullsized":"2026-02-20T21:46:20.092444+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:14:32.329895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010312629999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":828,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.092569+0000","last_change":"2026-02-20T21:05:32.885327+0000","last_active":"2026-02-20T21:46:20.092569+0000","last_peered":"2026-02-20T21:46:20.092569+0000","last_clean":"2026-02-20T21:46:20.092569+0000","last_became_active":"2026-02-20T21:05:30.882952+0000","last_became_peered":"2026-02-20T21:05:30.882952+0000","last_unstale":"2026-02-20T21:46:20.092569+0000","last_undegraded":"2026-02-20T21:46:20.092569+0000","last_fullsized":"2026-02-20T21:46:20.092569+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:32:38.163582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071972500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":829,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091185+0000","last_change":"2026-02-20T21:05:32.885072+0000","last_active":"2026-02-20T21:46:20.091185+0000","last_peered":"2026-02-20T21:46:20.091185+0000","last_clean":"2026-02-20T21:46:20.091185+0000","last_became_active":"2026-02-20T21:05:30.883044+0000","last_became_peered":"2026-02-20T21:05:30.883044+0000","last_unstale":"2026-02-20T21:46:20.091185+0000","last_undegraded":"2026-02-20T21:46:20.091185+0000","last_fullsized":"2026-02-20T21:46:20.091185+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:57:08.905691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055306500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":829,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.090983+0000","last_change":"2026-02-20T21:05:32.885039+0000","last_active":"2026-02-20T21:46:20.090983+0000","last_peered":"2026-02-20T21:46:20.090983+0000","last_clean":"2026-02-20T21:46:20.090983+0000","last_became_active":"2026-02-20T21:05:30.883115+0000","last_became_peered":"2026-02-20T21:05:30.883115+0000","last_unstale":"2026-02-20T21:46:20.090983+0000","last_undegraded":"2026-02-20T21:46:20.090983+0000","last_fullsized":"2026-02-20T21:46:20.090983+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:11:05.945358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052496400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":831,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091395+0000","last_change":"2026-02-20T21:05:32.883927+0000","last_active":"2026-02-20T21:46:20.091395+0000","last_peered":"2026-02-20T21:46:20.091395+0000","last_clean":"2026-02-20T21:46:20.091395+0000","last_became_active":"2026-02-20T21:05:30.883032+0000","last_became_peered":"2026-02-20T21:05:30.883032+0000","last_unstale":"2026-02-20T21:46:20.091395+0000","last_undegraded":"2026-02-20T21:46:20.091395+0000","last_fullsized":"2026-02-20T21:46:20.091395+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:07:25.195375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077015799999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":827,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091171+0000","last_change":"2026-02-20T21:05:32.927062+0000","last_active":"2026-02-20T21:46:20.091171+0000","last_peered":"2026-02-20T21:46:20.091171+0000","last_clean":"2026-02-20T21:46:20.091171+0000","last_became_active":"2026-02-20T21:05:30.883679+0000","last_became_peered":"2026-02-20T21:05:30.883679+0000","last_unstale":"2026-02-20T21:46:20.091171+0000","last_undegraded":"2026-02-20T21:46:20.091171+0000","last_fullsized":"2026-02-20T21:46:20.091171+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:43:48.880973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021062000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":828,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091655+0000","last_change":"2026-02-20T21:05:32.927079+0000","last_active":"2026-02-20T21:46:20.091655+0000","last_peered":"2026-02-20T21:46:20.091655+0000","last_clean":"2026-02-20T21:46:20.091655+0000","last_became_active":"2026-02-20T21:05:30.883572+0000","last_became_peered":"2026-02-20T21:05:30.883572+0000","last_unstale":"2026-02-20T21:46:20.091655+0000","last_undegraded":"2026-02-20T21:46:20.091655+0000","last_fullsized":"2026-02-20T21:46:20.091655+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:29:24.098186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022568800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":830,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.092509+0000","last_change":"2026-02-20T21:05:32.885282+0000","last_active":"2026-02-20T21:46:20.092509+0000","last_peered":"2026-02-20T21:46:20.092509+0000","last_clean":"2026-02-20T21:46:20.092509+0000","last_became_active":"2026-02-20T21:05:30.882188+0000","last_became_peered":"2026-02-20T21:05:30.882188+0000","last_unstale":"2026-02-20T21:46:20.092509+0000","last_undegraded":"2026-02-20T21:46:20.092509+0000","last_fullsized":"2026-02-20T21:46:20.092509+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:59:42.030633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075006200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":921,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091293+0000","last_change":"2026-02-20T21:05:28.168843+0000","last_active":"2026-02-20T21:46:20.091293+0000","last_peered":"2026-02-20T21:46:20.091293+0000","last_clean":"2026-02-20T21:46:20.091293+0000","last_became_active":"2026-02-20T21:05:28.168224+0000","last_became_peered":"2026-02-20T21:05:28.168224+0000","last_unstale":"2026-02-20T21:46:20.091293+0000","last_undegraded":"2026-02-20T21:46:20.091293+0000","last_fullsized":"2026-02-20T21:46:20.091293+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:26.870888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:26.870888+0000","last_clean_scrub_stamp":"2026-02-20T21:05:26.870888+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:36:01.610328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738862,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1404332,"kb_used_data":4764,"kb_used_omap":94,"kb_used_meta":1399457,"kb_avail":92967508,"statfs":{"total":96636764160,"available":95198728192,"internally_reserved":0,"allocated":4878336,"data_stored":2255193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":96524,"internal_metadata":1433044724},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738863,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2162564,"kb_used_data":276916,"kb_used_omap":313,"kb_used_meta":1885318,"kb_avail":92209276,"statfs":{"total":96636764160,"available":94422298624,"internally_reserved":0,"allocated":283561984,"data_stored":280452985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":321013,"internal_metadata":1930566155},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738863,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1614188,"kb_used_data":338012,"kb_used_omap":151,"kb_used_meta":1276008,"kb_avail":92757652,"statfs":{"total":96636764160,"available":94983835648,"internally_reserved":0,"allocated":346124288,"data_stored":342948745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":155064,"internal_metadata":1306632776},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:46:26.496 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-20T21:46:26.496 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:46:26.677 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:46:26.677 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-20T21:46:26.686 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":1641,"stamp":"2026-02-20T21:46:25.097161+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":5181084,"kb_used_data":619692,"kb_used_omap":559,"kb_used_meta":4560784,"kb_avail":277934436,"statfs":{"total":289910292480,"available":284604862464,"internally_reserved":0,"allocated":634564608,"data_stored":625656923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":572601,"internal_metadata":4670243655},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-1383614684,"num_objects":-226,"num_object_clones":0,"num_object_copies":-452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-226,"num_whiteouts":0,"num_read":-2398110,"num_read_kb":-89552721,"num_write":-2345210,"num_write_kb":-91560408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.120587"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":828,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.092444+0000","last_change":"2026-02-20T21:05:32.885706+0000","last_active":"2026-02-20T21:46:20.092444+0000","last_peered":"2026-02-20T21:46:20.092444+0000","last_clean":"2026-02-20T21:46:20.092444+0000","last_became_active":"2026-02-20T21:05:30.883104+0000","last_became_peered":"2026-02-20T21:05:30.883104+0000","last_unstale":"2026-02-20T21:46:20.092444+0000","last_undegraded":"2026-02-20T21:46:20.092444+0000","last_fullsized":"2026-02-20T21:46:20.092444+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:14:32.329895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010312629999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":828,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.092569+0000","last_change":"2026-02-20T21:05:32.885327+0000","last_active":"2026-02-20T21:46:20.092569+0000","last_peered":"2026-02-20T21:46:20.092569+0000","last_clean":"2026-02-20T21:46:20.092569+0000","last_became_active":"2026-02-20T21:05:30.882952+0000","last_became_peered":"2026-02-20T21:05:30.882952+0000","last_unstale":"2026-02-20T21:46:20.092569+0000","last_undegraded":"2026-02-20T21:46:20.092569+0000","last_fullsized":"2026-02-20T21:46:20.092569+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:32:38.163582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071972500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":829,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091185+0000","last_change":"2026-02-20T21:05:32.885072+0000","last_active":"2026-02-20T21:46:20.091185+0000","last_peered":"2026-02-20T21:46:20.091185+0000","last_clean":"2026-02-20T21:46:20.091185+0000","last_became_active":"2026-02-20T21:05:30.883044+0000","last_became_peered":"2026-02-20T21:05:30.883044+0000","last_unstale":"2026-02-20T21:46:20.091185+0000","last_undegraded":"2026-02-20T21:46:20.091185+0000","last_fullsized":"2026-02-20T21:46:20.091185+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:57:08.905691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055306500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":829,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.090983+0000","last_change":"2026-02-20T21:05:32.885039+0000","last_active":"2026-02-20T21:46:20.090983+0000","last_peered":"2026-02-20T21:46:20.090983+0000","last_clean":"2026-02-20T21:46:20.090983+0000","last_became_active":"2026-02-20T21:05:30.883115+0000","last_became_peered":"2026-02-20T21:05:30.883115+0000","last_unstale":"2026-02-20T21:46:20.090983+0000","last_undegraded":"2026-02-20T21:46:20.090983+0000","last_fullsized":"2026-02-20T21:46:20.090983+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:11:05.945358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052496400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":831,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091395+0000","last_change":"2026-02-20T21:05:32.883927+0000","last_active":"2026-02-20T21:46:20.091395+0000","last_peered":"2026-02-20T21:46:20.091395+0000","last_clean":"2026-02-20T21:46:20.091395+0000","last_became_active":"2026-02-20T21:05:30.883032+0000","last_became_peered":"2026-02-20T21:05:30.883032+0000","last_unstale":"2026-02-20T21:46:20.091395+0000","last_undegraded":"2026-02-20T21:46:20.091395+0000","last_fullsized":"2026-02-20T21:46:20.091395+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:07:25.195375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077015799999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":827,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091171+0000","last_change":"2026-02-20T21:05:32.927062+0000","last_active":"2026-02-20T21:46:20.091171+0000","last_peered":"2026-02-20T21:46:20.091171+0000","last_clean":"2026-02-20T21:46:20.091171+0000","last_became_active":"2026-02-20T21:05:30.883679+0000","last_became_peered":"2026-02-20T21:05:30.883679+0000","last_unstale":"2026-02-20T21:46:20.091171+0000","last_undegraded":"2026-02-20T21:46:20.091171+0000","last_fullsized":"2026-02-20T21:46:20.091171+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:43:48.880973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021062000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":828,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091655+0000","last_change":"2026-02-20T21:05:32.927079+0000","last_active":"2026-02-20T21:46:20.091655+0000","last_peered":"2026-02-20T21:46:20.091655+0000","last_clean":"2026-02-20T21:46:20.091655+0000","last_became_active":"2026-02-20T21:05:30.883572+0000","last_became_peered":"2026-02-20T21:05:30.883572+0000","last_unstale":"2026-02-20T21:46:20.091655+0000","last_undegraded":"2026-02-20T21:46:20.091655+0000","last_fullsized":"2026-02-20T21:46:20.091655+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:29:24.098186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022568800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":830,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.092509+0000","last_change":"2026-02-20T21:05:32.885282+0000","last_active":"2026-02-20T21:46:20.092509+0000","last_peered":"2026-02-20T21:46:20.092509+0000","last_clean":"2026-02-20T21:46:20.092509+0000","last_became_active":"2026-02-20T21:05:30.882188+0000","last_became_peered":"2026-02-20T21:05:30.882188+0000","last_unstale":"2026-02-20T21:46:20.092509+0000","last_undegraded":"2026-02-20T21:46:20.092509+0000","last_fullsized":"2026-02-20T21:46:20.092509+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:59:42.030633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075006200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":921,"reported_epoch":411,"state":"active+clean","last_fresh":"2026-02-20T21:46:20.091293+0000","last_change":"2026-02-20T21:05:28.168843+0000","last_active":"2026-02-20T21:46:20.091293+0000","last_peered":"2026-02-20T21:46:20.091293+0000","last_clean":"2026-02-20T21:46:20.091293+0000","last_became_active":"2026-02-20T21:05:28.168224+0000","last_became_peered":"2026-02-20T21:05:28.168224+0000","last_unstale":"2026-02-20T21:46:20.091293+0000","last_undegraded":"2026-02-20T21:46:20.091293+0000","last_fullsized":"2026-02-20T21:46:20.091293+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:26.870888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:26.870888+0000","last_clean_scrub_stamp":"2026-02-20T21:05:26.870888+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:36:01.610328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738862,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1404332,"kb_used_data":4764,"kb_used_omap":94,"kb_used_meta":1399457,"kb_avail":92967508,"statfs":{"total":96636764160,"available":95198728192,"internally_reserved":0,"allocated":4878336,"data_stored":2255193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":96524,"internal_metadata":1433044724},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738863,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2162564,"kb_used_data":276916,"kb_used_omap":313,"kb_used_meta":1885318,"kb_avail":92209276,"statfs":{"total":96636764160,"available":94422298624,"internally_reserved":0,"allocated":283561984,"data_stored":280452985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":321013,"internal_metadata":1930566155},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738863,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1614188,"kb_used_data":338012,"kb_used_omap":151,"kb_used_meta":1276008,"kb_avail":92757652,"statfs":{"total":96636764160,"available":94983835648,"internally_reserved":0,"allocated":346124288,"data_stored":342948745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":155064,"internal_metadata":1306632776},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:46:26.687 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-20T21:46:26.870 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:46:26.871 INFO:teuthology.orchestra.run.trial163.stdout:{"epoch":413,"fsid":"6b7cbdea-dd31-4132-aa0e-90dc10bb8129","created":"2026-02-20T21:05:20.858799+0000","modified":"2026-02-20T21:46:25.094986+0000","last_up_change":"2026-02-20T21:05:24.866038+0000","last_in_change":"2026-02-20T21:05:21.536084+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:05:26.567416+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:05:28.922053+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"956a0026-bd9c-4ea8-b787-0eb4724015b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6808","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6809","nonce":2921121942}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6810","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6811","nonce":2921121942}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6814","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6815","nonce":2921121942}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6812","nonce":2921121942},{"type":"v1","addr":"10.20.193.163:6813","nonce":2921121942}]},"public_addr":"10.20.193.163:6809/2921121942","cluster_addr":"10.20.193.163:6811/2921121942","heartbeat_back_addr":"10.20.193.163:6815/2921121942","heartbeat_front_addr":"10.20.193.163:6813/2921121942","state":["exists","up"]},{"osd":1,"uuid":"fc9a9b6f-73d6-4c05-a019-14d361d3a240","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":19,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6816","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6817","nonce":3447493578}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6818","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6819","nonce":3447493578}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6822","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6823","nonce":3447493578}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6820","nonce":3447493578},{"type":"v1","addr":"10.20.193.163:6821","nonce":3447493578}]},"public_addr":"10.20.193.163:6817/3447493578","cluster_addr":"10.20.193.163:6819/3447493578","heartbeat_back_addr":"10.20.193.163:6823/3447493578","heartbeat_front_addr":"10.20.193.163:6821/3447493578","state":["exists","up"]},{"osd":2,"uuid":"17e3c46a-bcf7-466e-a56c-33ba58b56170","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6800","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6801","nonce":3294996969}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6802","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6803","nonce":3294996969}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6806","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6807","nonce":3294996969}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.163:6804","nonce":3294996969},{"type":"v1","addr":"10.20.193.163:6805","nonce":3294996969}]},"public_addr":"10.20.193.163:6801/3294996969","cluster_addr":"10.20.193.163:6803/3294996969","heartbeat_back_addr":"10.20.193.163:6807/3294996969","heartbeat_front_addr":"10.20.193.163:6805/3294996969","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:23.323693+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:23.393980+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:23.304166+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"10.20.193.163:0/7470644":"2026-02-20T22:46:19.486430+0000","10.20.193.163:0/4129633343":"2026-02-20T22:45:34.722052+0000","10.20.193.163:0/344052385":"2026-02-20T22:44:56.267704+0000","10.20.193.163:0/2725621795":"2026-02-20T22:42:09.089807+0000","10.20.193.163:0/1938059020":"2026-02-20T22:41:25.004963+0000","10.20.193.163:0/446004721":"2026-02-20T22:40:33.302780+0000","10.20.193.163:0/538146431":"2026-02-20T22:39:50.396964+0000","10.20.193.163:0/440596746":"2026-02-20T22:37:58.486970+0000","10.20.193.163:0/1810234339":"2026-02-20T22:29:36.603023+0000","10.20.193.163:0/2335430242":"2026-02-20T22:29:10.071550+0000","10.20.193.163:0/4163408017":"2026-02-20T22:32:19.262087+0000","10.20.193.163:0/1044517540":"2026-02-20T22:34:33.558547+0000","10.20.193.163:0/1670388416":"2026-02-20T22:30:44.212737+0000","10.20.193.163:0/722651736":"2026-02-20T22:28:26.512979+0000","10.20.193.163:0/1093421476":"2026-02-20T22:28:13.859240+0000","10.20.193.163:0/70814225":"2026-02-20T22:35:11.593938+0000","10.20.193.163:0/4261287036":"2026-02-20T22:27:31.358352+0000","10.20.193.163:0/639876315":"2026-02-20T22:08:30.680144+0000","10.20.193.163:0/511131183":"2026-02-20T22:37:38.631871+0000","10.20.193.163:0/1642332849":"2026-02-20T22:30:23.500764+0000","10.20.193.163:0/1437210869":"2026-02-20T22:29:48.144731+0000","10.20.193.163:0/1952378880":"2026-02-20T22:44:13.513815+0000","10.20.193.163:0/3217279415":"2026-02-20T22:35:47.056345+0000","10.20.193.163:0/673190071":"2026-02-20T22:27:43.861827+0000","10.20.193.163:0/2968021999":"2026-02-20T22:30:12.911133+0000","10.20.193.163:0/2518112233":"2026-02-20T22:28:44.322859+0000","10.20.193.163:0/1685185127":"2026-02-20T22:10:03.361430+0000","10.20.193.163:0/2260611010":"2026-02-20T22:16:13.666388+0000","10.20.193.163:0/2417014694":"2026-02-20T22:12:18.952546+0000","10.20.193.163:0/2774597192":"2026-02-20T22:29:05.115610+0000","10.20.193.163:0/3057624106":"2026-02-20T22:29:22.748427+0000","10.20.193.163:0/3160177286":"2026-02-20T22:30:33.646258+0000","10.20.193.163:0/4238797292":"2026-02-20T22:34:01.648525+0000","10.20.193.163:0/936982967":"2026-02-20T22:33:03.152102+0000","10.20.193.163:0/3382628474":"2026-02-20T22:07:26.089205+0000","10.20.193.163:0/1027565600":"2026-02-20T22:37:22.956515+0000","10.20.193.163:0/1145564406":"2026-02-20T22:34:46.718273+0000","10.20.193.163:0/1841616037":"2026-02-20T22:17:31.709283+0000","10.20.193.163:0/2146152314":"2026-02-20T22:15:11.740833+0000","10.20.193.163:0/2637519019":"2026-02-20T22:28:55.082234+0000","10.20.193.163:0/3700024625":"2026-02-20T22:25:31.843719+0000","10.20.193.163:0/3218644643":"2026-02-20T22:36:16.191938+0000","10.20.193.163:0/2078385094":"2026-02-20T22:30:54.441181+0000","10.20.193.163:0/548633747":"2026-02-20T22:43:36.027078+0000","10.20.193.163:0/1513385489":"2026-02-20T22:36:01.457377+0000","10.20.193.163:0/805349561":"2026-02-20T22:31:03.611517+0000","10.20.193.163:0/3574735801":"2026-02-20T22:32:08.467748+0000","10.20.193.163:0/4217878788":"2026-02-20T22:27:52.091915+0000","10.20.193.163:0/1914137196":"2026-02-20T22:32:33.294334+0000","10.20.193.163:0/1527943034":"2026-02-20T22:37:10.147777+0000","10.20.193.163:0/1880056838":"2026-02-20T22:32:45.237377+0000","10.20.193.163:0/376487030":"2026-02-20T22:13:34.754834+0000","10.20.193.163:0/1455485871":"2026-02-20T22:32:55.833996+0000","10.20.193.163:0/259393584":"2026-02-20T22:32:23.969983+0000","10.20.193.163:0/2972921768":"2026-02-20T22:33:14.450797+0000","10.20.193.163:0/82868445":"2026-02-20T22:27:11.080821+0000","10.20.193.163:0/2005239249":"2026-02-20T22:34:12.349380+0000","10.20.193.163:0/3985864117":"2026-02-20T22:42:52.226505+0000","10.20.193.163:0/3038450277":"2026-02-20T22:34:26.634058+0000","10.20.193.163:0/1870790257":"2026-02-20T22:27:21.335835+0000","10.20.193.163:0/2904792067":"2026-02-20T22:34:58.760184+0000","10.20.193.163:0/1149035289":"2026-02-20T22:35:24.043607+0000","10.20.193.163:0/1801628472":"2026-02-20T22:31:15.657361+0000","10.20.193.163:0/3380190225":"2026-02-20T22:36:11.078879+0000","10.20.193.163:0/2304650994":"2026-02-20T22:29:59.609234+0000","10.20.193.163:0/3282383221":"2026-02-20T22:14:20.714243+0000","10.20.193.163:0/4266543747":"2026-02-20T22:36:57.689484+0000","10.20.193.163:0/3485145081":"2026-02-20T22:11:11.037149+0000","10.20.193.163:0/1218771585":"2026-02-20T22:37:32.353316+0000","10.20.193.163:0/593453047":"2026-02-20T22:37:48.696635+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:46:27.882 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-20T21:46:27.882 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-20T21:46:27.966 INFO:teuthology.orchestra.run.trial163.stdout:{ 2026-02-20T21:46:27.967 INFO:teuthology.orchestra.run.trial163.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T21:46:27.967 INFO:teuthology.orchestra.run.trial163.stdout:} 2026-02-20T21:46:27.974 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-20T21:46:28.154 INFO:teuthology.orchestra.run.trial163.stderr:instructed osd(s) 0 to deep-scrub 2026-02-20T21:46:28.166 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-20T21:46:28.166 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-20T21:46:28.249 INFO:teuthology.orchestra.run.trial163.stdout:{ 2026-02-20T21:46:28.249 INFO:teuthology.orchestra.run.trial163.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T21:46:28.249 INFO:teuthology.orchestra.run.trial163.stdout:} 2026-02-20T21:46:28.254 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-20T21:46:28.437 INFO:teuthology.orchestra.run.trial163.stderr:instructed osd(s) 1 to deep-scrub 2026-02-20T21:46:28.446 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-20T21:46:28.446 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-20T21:46:28.527 INFO:teuthology.orchestra.run.trial163.stdout:{ 2026-02-20T21:46:28.527 INFO:teuthology.orchestra.run.trial163.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T21:46:28.527 INFO:teuthology.orchestra.run.trial163.stdout:} 2026-02-20T21:46:28.533 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-20T21:46:28.714 INFO:teuthology.orchestra.run.trial163.stderr:instructed osd(s) 2 to deep-scrub 2026-02-20T21:46:28.723 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:46:28.905 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:46:28.906 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-20T21:46:28.915 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":1642,"stamp":"2026-02-20T21:46:26.977511+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":185,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4613020,"kb_used_data":15532,"kb_used_omap":565,"kb_used_meta":4596874,"kb_avail":278502500,"statfs":{"total":289910292480,"available":285186560000,"internally_reserved":0,"allocated":15904768,"data_stored":7987522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":578813,"internal_metadata":4707199747},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-1383614684,"num_objects":-226,"num_object_clones":0,"num_object_copies":-452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-226,"num_whiteouts":0,"num_read":-2398110,"num_read_kb":-89552721,"num_write":-2345210,"num_write_kb":-91560408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-19,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.887952"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":832,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:25.099674+0000","last_change":"2026-02-20T21:05:32.885706+0000","last_active":"2026-02-20T21:46:25.099674+0000","last_peered":"2026-02-20T21:46:25.099674+0000","last_clean":"2026-02-20T21:46:25.099674+0000","last_became_active":"2026-02-20T21:05:30.883104+0000","last_became_peered":"2026-02-20T21:05:30.883104+0000","last_unstale":"2026-02-20T21:46:25.099674+0000","last_undegraded":"2026-02-20T21:46:25.099674+0000","last_fullsized":"2026-02-20T21:46:25.099674+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:14:32.329895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010312629999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":832,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:25.099338+0000","last_change":"2026-02-20T21:05:32.885327+0000","last_active":"2026-02-20T21:46:25.099338+0000","last_peered":"2026-02-20T21:46:25.099338+0000","last_clean":"2026-02-20T21:46:25.099338+0000","last_became_active":"2026-02-20T21:05:30.882952+0000","last_became_peered":"2026-02-20T21:05:30.882952+0000","last_unstale":"2026-02-20T21:46:25.099338+0000","last_undegraded":"2026-02-20T21:46:25.099338+0000","last_fullsized":"2026-02-20T21:46:25.099338+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:32:38.163582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071972500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":833,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:25.098679+0000","last_change":"2026-02-20T21:05:32.885072+0000","last_active":"2026-02-20T21:46:25.098679+0000","last_peered":"2026-02-20T21:46:25.098679+0000","last_clean":"2026-02-20T21:46:25.098679+0000","last_became_active":"2026-02-20T21:05:30.883044+0000","last_became_peered":"2026-02-20T21:05:30.883044+0000","last_unstale":"2026-02-20T21:46:25.098679+0000","last_undegraded":"2026-02-20T21:46:25.098679+0000","last_fullsized":"2026-02-20T21:46:25.098679+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:57:08.905691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055306500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":833,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:25.099653+0000","last_change":"2026-02-20T21:05:32.885039+0000","last_active":"2026-02-20T21:46:25.099653+0000","last_peered":"2026-02-20T21:46:25.099653+0000","last_clean":"2026-02-20T21:46:25.099653+0000","last_became_active":"2026-02-20T21:05:30.883115+0000","last_became_peered":"2026-02-20T21:05:30.883115+0000","last_unstale":"2026-02-20T21:46:25.099653+0000","last_undegraded":"2026-02-20T21:46:25.099653+0000","last_fullsized":"2026-02-20T21:46:25.099653+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T23:11:05.945358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052496400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":835,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:25.099733+0000","last_change":"2026-02-20T21:05:32.883927+0000","last_active":"2026-02-20T21:46:25.099733+0000","last_peered":"2026-02-20T21:46:25.099733+0000","last_clean":"2026-02-20T21:46:25.099733+0000","last_became_active":"2026-02-20T21:05:30.883032+0000","last_became_peered":"2026-02-20T21:05:30.883032+0000","last_unstale":"2026-02-20T21:46:25.099733+0000","last_undegraded":"2026-02-20T21:46:25.099733+0000","last_fullsized":"2026-02-20T21:46:25.099733+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:07:25.195375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077015799999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":831,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:25.099020+0000","last_change":"2026-02-20T21:05:32.927062+0000","last_active":"2026-02-20T21:46:25.099020+0000","last_peered":"2026-02-20T21:46:25.099020+0000","last_clean":"2026-02-20T21:46:25.099020+0000","last_became_active":"2026-02-20T21:05:30.883679+0000","last_became_peered":"2026-02-20T21:05:30.883679+0000","last_unstale":"2026-02-20T21:46:25.099020+0000","last_undegraded":"2026-02-20T21:46:25.099020+0000","last_fullsized":"2026-02-20T21:46:25.099020+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:43:48.880973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021062000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":832,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:25.098992+0000","last_change":"2026-02-20T21:05:32.927079+0000","last_active":"2026-02-20T21:46:25.098992+0000","last_peered":"2026-02-20T21:46:25.098992+0000","last_clean":"2026-02-20T21:46:25.098992+0000","last_became_active":"2026-02-20T21:05:30.883572+0000","last_became_peered":"2026-02-20T21:05:30.883572+0000","last_unstale":"2026-02-20T21:46:25.098992+0000","last_undegraded":"2026-02-20T21:46:25.098992+0000","last_fullsized":"2026-02-20T21:46:25.098992+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:29:24.098186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022568800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":834,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:25.099860+0000","last_change":"2026-02-20T21:05:32.885282+0000","last_active":"2026-02-20T21:46:25.099860+0000","last_peered":"2026-02-20T21:46:25.099860+0000","last_clean":"2026-02-20T21:46:25.099860+0000","last_became_active":"2026-02-20T21:05:30.882188+0000","last_became_peered":"2026-02-20T21:05:30.882188+0000","last_unstale":"2026-02-20T21:46:25.099860+0000","last_undegraded":"2026-02-20T21:46:25.099860+0000","last_fullsized":"2026-02-20T21:46:25.099860+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:29.875649+0000","last_clean_scrub_stamp":"2026-02-20T21:05:29.875649+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:59:42.030633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075006200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":925,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:25.100761+0000","last_change":"2026-02-20T21:05:28.168843+0000","last_active":"2026-02-20T21:46:25.100761+0000","last_peered":"2026-02-20T21:46:25.100761+0000","last_clean":"2026-02-20T21:46:25.100761+0000","last_became_active":"2026-02-20T21:05:28.168224+0000","last_became_peered":"2026-02-20T21:05:28.168224+0000","last_unstale":"2026-02-20T21:46:25.100761+0000","last_undegraded":"2026-02-20T21:46:25.100761+0000","last_fullsized":"2026-02-20T21:46:25.100761+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:26.870888+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:26.870888+0000","last_clean_scrub_stamp":"2026-02-20T21:05:26.870888+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:36:01.610328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738863,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1422404,"kb_used_data":4788,"kb_used_omap":92,"kb_used_meta":1417507,"kb_avail":92949436,"statfs":{"total":96636764160,"available":95180222464,"internally_reserved":0,"allocated":4902912,"data_stored":2268916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":95200,"internal_metadata":1451527200},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738864,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1909068,"kb_used_data":5372,"kb_used_omap":316,"kb_used_meta":1903363,"kb_avail":92462772,"statfs":{"total":96636764160,"available":94681878528,"internally_reserved":0,"allocated":5500928,"data_stored":2859303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":324045,"internal_metadata":1949044275},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738864,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1281548,"kb_used_data":5372,"kb_used_omap":155,"kb_used_meta":1276004,"kb_avail":93090292,"statfs":{"total":96636764160,"available":95324459008,"internally_reserved":0,"allocated":5500928,"data_stored":2859303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":159568,"internal_metadata":1306628272},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:46:28.920 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-20T21:05:29.875649+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=29, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:28.920 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-20T21:05:29.875649+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=29, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:28.920 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-20T21:05:29.875649+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=29, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:28.920 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-20T21:05:29.875649+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=29, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:28.921 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-20T21:05:29.875649+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=29, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:28.921 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-20T21:05:29.875649+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=29, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:28.921 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-20T21:05:29.875649+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=29, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:28.921 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-20T21:05:29.875649+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=29, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:28.921 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-20T21:05:26.870888+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=26, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:28.921 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-20T21:46:48.923 DEBUG:teuthology.orchestra.run.trial163:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-20T21:46:49.105 INFO:teuthology.orchestra.run.trial163.stdout: 2026-02-20T21:46:49.106 INFO:teuthology.orchestra.run.trial163.stderr:dumped all 2026-02-20T21:46:49.115 INFO:teuthology.orchestra.run.trial163.stdout:{"pg_ready":true,"pg_map":{"version":1653,"stamp":"2026-02-20T21:46:48.979334+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4613020,"kb_used_data":15532,"kb_used_omap":565,"kb_used_meta":4596874,"kb_avail":278502500,"statfs":{"total":289910292480,"available":285186560000,"internally_reserved":0,"allocated":15904768,"data_stored":7987522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":578813,"internal_metadata":4707199747},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000936"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":841,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:34.416041+0000","last_change":"2026-02-20T21:46:34.415991+0000","last_active":"2026-02-20T21:46:34.416041+0000","last_peered":"2026-02-20T21:46:34.416041+0000","last_clean":"2026-02-20T21:46:34.416041+0000","last_became_active":"2026-02-20T21:05:30.883104+0000","last_became_peered":"2026-02-20T21:05:30.883104+0000","last_unstale":"2026-02-20T21:46:34.416041+0000","last_undegraded":"2026-02-20T21:46:34.416041+0000","last_fullsized":"2026-02-20T21:46:34.416041+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:46:34.415932+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:46:34.415932+0000","last_clean_scrub_stamp":"2026-02-20T21:46:34.415932+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:02:42.055948+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.0010312629999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":841,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:32.412605+0000","last_change":"2026-02-20T21:46:32.412584+0000","last_active":"2026-02-20T21:46:32.412605+0000","last_peered":"2026-02-20T21:46:32.412605+0000","last_clean":"2026-02-20T21:46:32.412605+0000","last_became_active":"2026-02-20T21:05:30.882952+0000","last_became_peered":"2026-02-20T21:05:30.882952+0000","last_unstale":"2026-02-20T21:46:32.412605+0000","last_undegraded":"2026-02-20T21:46:32.412605+0000","last_fullsized":"2026-02-20T21:46:32.412605+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:46:32.412543+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:46:32.412543+0000","last_clean_scrub_stamp":"2026-02-20T21:46:32.412543+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:37:21.389081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00071972500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":841,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:33.430534+0000","last_change":"2026-02-20T21:46:33.430534+0000","last_active":"2026-02-20T21:46:33.430534+0000","last_peered":"2026-02-20T21:46:33.430534+0000","last_clean":"2026-02-20T21:46:33.430534+0000","last_became_active":"2026-02-20T21:05:30.883044+0000","last_became_peered":"2026-02-20T21:05:30.883044+0000","last_unstale":"2026-02-20T21:46:33.430534+0000","last_undegraded":"2026-02-20T21:46:33.430534+0000","last_fullsized":"2026-02-20T21:46:33.430534+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:46:33.430459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:46:33.430459+0000","last_clean_scrub_stamp":"2026-02-20T21:46:33.430459+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:37:55.591973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055306500000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":841,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:31.406528+0000","last_change":"2026-02-20T21:46:31.406528+0000","last_active":"2026-02-20T21:46:31.406528+0000","last_peered":"2026-02-20T21:46:31.406528+0000","last_clean":"2026-02-20T21:46:31.406528+0000","last_became_active":"2026-02-20T21:05:30.883115+0000","last_became_peered":"2026-02-20T21:05:30.883115+0000","last_unstale":"2026-02-20T21:46:31.406528+0000","last_undegraded":"2026-02-20T21:46:31.406528+0000","last_fullsized":"2026-02-20T21:46:31.406528+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:46:31.406475+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:46:31.406475+0000","last_clean_scrub_stamp":"2026-02-20T21:46:31.406475+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:24:40.593934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00052496400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"413'3","reported_seq":844,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:28.373897+0000","last_change":"2026-02-20T21:46:28.373897+0000","last_active":"2026-02-20T21:46:28.373897+0000","last_peered":"2026-02-20T21:46:28.373897+0000","last_clean":"2026-02-20T21:46:28.373897+0000","last_became_active":"2026-02-20T21:05:30.883032+0000","last_became_peered":"2026-02-20T21:05:30.883032+0000","last_unstale":"2026-02-20T21:46:28.373897+0000","last_undegraded":"2026-02-20T21:46:28.373897+0000","last_fullsized":"2026-02-20T21:46:28.373897+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"413'3","last_scrub_stamp":"2026-02-20T21:46:28.373833+0000","last_deep_scrub":"413'3","last_deep_scrub_stamp":"2026-02-20T21:46:28.373833+0000","last_clean_scrub_stamp":"2026-02-20T21:46:28.373833+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:33:28.562927+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00077015799999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":839,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:29.600384+0000","last_change":"2026-02-20T21:46:29.600384+0000","last_active":"2026-02-20T21:46:29.600384+0000","last_peered":"2026-02-20T21:46:29.600384+0000","last_clean":"2026-02-20T21:46:29.600384+0000","last_became_active":"2026-02-20T21:05:30.883679+0000","last_became_peered":"2026-02-20T21:05:30.883679+0000","last_unstale":"2026-02-20T21:46:29.600384+0000","last_undegraded":"2026-02-20T21:46:29.600384+0000","last_fullsized":"2026-02-20T21:46:29.600384+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:46:29.600325+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:46:29.600325+0000","last_clean_scrub_stamp":"2026-02-20T21:46:29.600325+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:26:05.175728+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00021062000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":840,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:30.636282+0000","last_change":"2026-02-20T21:46:30.636282+0000","last_active":"2026-02-20T21:46:30.636282+0000","last_peered":"2026-02-20T21:46:30.636282+0000","last_clean":"2026-02-20T21:46:30.636282+0000","last_became_active":"2026-02-20T21:05:30.883572+0000","last_became_peered":"2026-02-20T21:05:30.883572+0000","last_unstale":"2026-02-20T21:46:30.636282+0000","last_undegraded":"2026-02-20T21:46:30.636282+0000","last_fullsized":"2026-02-20T21:46:30.636282+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:46:30.636223+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:46:30.636223+0000","last_clean_scrub_stamp":"2026-02-20T21:46:30.636223+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:23:27.245783+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00022568800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"413'2","reported_seq":843,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:30.430315+0000","last_change":"2026-02-20T21:46:30.430315+0000","last_active":"2026-02-20T21:46:30.430315+0000","last_peered":"2026-02-20T21:46:30.430315+0000","last_clean":"2026-02-20T21:46:30.430315+0000","last_became_active":"2026-02-20T21:05:30.882188+0000","last_became_peered":"2026-02-20T21:05:30.882188+0000","last_unstale":"2026-02-20T21:46:30.430315+0000","last_undegraded":"2026-02-20T21:46:30.430315+0000","last_fullsized":"2026-02-20T21:46:30.430315+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"413'2","last_scrub_stamp":"2026-02-20T21:46:30.430269+0000","last_deep_scrub":"413'2","last_deep_scrub_stamp":"2026-02-20T21:46:30.430269+0000","last_clean_scrub_stamp":"2026-02-20T21:46:30.430269+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:59:37.942205+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.00075006200000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"413'74","reported_seq":935,"reported_epoch":413,"state":"active+clean","last_fresh":"2026-02-20T21:46:29.444812+0000","last_change":"2026-02-20T21:46:29.444812+0000","last_active":"2026-02-20T21:46:29.444812+0000","last_peered":"2026-02-20T21:46:29.444812+0000","last_clean":"2026-02-20T21:46:29.444812+0000","last_became_active":"2026-02-20T21:05:28.168224+0000","last_became_peered":"2026-02-20T21:05:28.168224+0000","last_unstale":"2026-02-20T21:46:29.444812+0000","last_undegraded":"2026-02-20T21:46:29.444812+0000","last_fullsized":"2026-02-20T21:46:29.444812+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"413'74","last_scrub_stamp":"2026-02-20T21:46:29.444753+0000","last_deep_scrub":"413'74","last_deep_scrub_stamp":"2026-02-20T21:46:29.444753+0000","last_clean_scrub_stamp":"2026-02-20T21:46:29.444753+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T07:24:53.733458+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738867,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1422404,"kb_used_data":4788,"kb_used_omap":92,"kb_used_meta":1417507,"kb_avail":92949436,"statfs":{"total":96636764160,"available":95180222464,"internally_reserved":0,"allocated":4902912,"data_stored":2268916,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":95200,"internal_metadata":1451527200},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738868,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1909068,"kb_used_data":5372,"kb_used_omap":316,"kb_used_meta":1903363,"kb_avail":92462772,"statfs":{"total":96636764160,"available":94681878528,"internally_reserved":0,"allocated":5500928,"data_stored":2859303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":324045,"internal_metadata":1949044275},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738868,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1281548,"kb_used_data":5372,"kb_used_omap":155,"kb_used_meta":1276004,"kb_avail":93090292,"statfs":{"total":96636764160,"available":95324459008,"internally_reserved":0,"allocated":5500928,"data_stored":2859303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":159568,"internal_metadata":1306628272},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:46:49.116 DEBUG:teuthology.orchestra.run.trial163:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-20T21:46:49.309 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-20T21:46:49.309 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-20T21:46:49.310 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-20T21:46:49.310 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:49.397 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-20T21:46:49.397 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-20T21:46:49.397 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:49.463 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-20T21:46:49.464 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-20T21:46:49.464 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:49.542 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-20T21:46:49.542 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-20T21:46:49.543 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-20T21:46:49.543 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:49.578 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-20T21:46:49.578 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-20T21:46:49.579 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-20T21:46:49.579 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:46:49.592 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-20T21:46:49.592 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-20T21:46:49.592 DEBUG:teuthology.orchestra.run.trial163:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-20T21:46:49.658 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial163.front.sepia.ceph.com 2026-02-20T21:46:49.658 DEBUG:teuthology.orchestra.run.trial163:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-20T21:46:49.819 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial163.front.sepia.ceph.com 2026-02-20T21:46:49.820 DEBUG:teuthology.orchestra.run.trial163:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-20T21:46:49.971 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial163.front.sepia.ceph.com 2026-02-20T21:46:49.971 DEBUG:teuthology.orchestra.run.trial163:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-20T21:46:50.103 INFO:tasks.ceph:Archiving mon data... 2026-02-20T21:46:50.105 DEBUG:teuthology.misc:Transferring archived files from trial163:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424/data/cluster2.mon.a.tgz 2026-02-20T21:46:50.105 DEBUG:teuthology.orchestra.run.trial163:> mktemp 2026-02-20T21:46:50.123 INFO:teuthology.orchestra.run.trial163.stdout:/tmp/tmp.YH7H2KVK9a 2026-02-20T21:46:50.123 DEBUG:teuthology.orchestra.run.trial163:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.YH7H2KVK9a 2026-02-20T21:46:50.250 DEBUG:teuthology.orchestra.run.trial163:> sudo chmod 0666 /tmp/tmp.YH7H2KVK9a 2026-02-20T21:46:50.333 DEBUG:teuthology.orchestra.remote:trial163:/tmp/tmp.YH7H2KVK9a is 492KB 2026-02-20T21:46:50.434 DEBUG:teuthology.orchestra.run.trial163:> rm -fr /tmp/tmp.YH7H2KVK9a 2026-02-20T21:46:50.451 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-20T21:46:50.452 DEBUG:teuthology.orchestra.run.trial069:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-20T21:46:50.455 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-20T21:46:50.518 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-20T21:46:50.542 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-20T21:46:50.543 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:46:50.728 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:46:50.728 INFO:teuthology.orchestra.run.trial069.stderr:dumped all 2026-02-20T21:46:50.739 INFO:teuthology.orchestra.run.trial069.stdout:{"pg_ready":true,"pg_map":{"version":1584,"stamp":"2026-02-20T21:46:49.427276+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4380928,"kb_used_data":9232,"kb_used_omap":3139,"kb_used_meta":4368508,"kb_avail":278734592,"statfs":{"total":289910292480,"available":285424222208,"internally_reserved":0,"allocated":9453568,"data_stored":5550436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3214450,"internal_metadata":4473353102},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000988"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.609366+0000","last_change":"2026-02-20T21:05:10.057921+0000","last_active":"2026-02-20T21:46:24.609366+0000","last_peered":"2026-02-20T21:46:24.609366+0000","last_clean":"2026-02-20T21:46:24.609366+0000","last_became_active":"2026-02-20T21:05:08.055580+0000","last_became_peered":"2026-02-20T21:05:08.055580+0000","last_unstale":"2026-02-20T21:46:24.609366+0000","last_undegraded":"2026-02-20T21:46:24.609366+0000","last_fullsized":"2026-02-20T21:46:24.609366+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:02:05.883424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079302400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.610039+0000","last_change":"2026-02-20T21:05:10.057723+0000","last_active":"2026-02-20T21:46:24.610039+0000","last_peered":"2026-02-20T21:46:24.610039+0000","last_clean":"2026-02-20T21:46:24.610039+0000","last_became_active":"2026-02-20T21:05:08.055678+0000","last_became_peered":"2026-02-20T21:05:08.055678+0000","last_unstale":"2026-02-20T21:46:24.610039+0000","last_undegraded":"2026-02-20T21:46:24.610039+0000","last_fullsized":"2026-02-20T21:46:24.610039+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:13:21.229779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061507100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.609652+0000","last_change":"2026-02-20T21:05:10.057558+0000","last_active":"2026-02-20T21:46:24.609652+0000","last_peered":"2026-02-20T21:46:24.609652+0000","last_clean":"2026-02-20T21:46:24.609652+0000","last_became_active":"2026-02-20T21:05:08.055784+0000","last_became_peered":"2026-02-20T21:05:08.055784+0000","last_unstale":"2026-02-20T21:46:24.609652+0000","last_undegraded":"2026-02-20T21:46:24.609652+0000","last_fullsized":"2026-02-20T21:46:24.609652+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:08:45.110689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046839599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.608827+0000","last_change":"2026-02-20T21:05:10.058070+0000","last_active":"2026-02-20T21:46:24.608827+0000","last_peered":"2026-02-20T21:46:24.608827+0000","last_clean":"2026-02-20T21:46:24.608827+0000","last_became_active":"2026-02-20T21:05:08.055399+0000","last_became_peered":"2026-02-20T21:05:08.055399+0000","last_unstale":"2026-02-20T21:46:24.608827+0000","last_undegraded":"2026-02-20T21:46:24.608827+0000","last_fullsized":"2026-02-20T21:46:24.608827+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:30:58.729968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091594400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":680,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.051709+0000","last_change":"2026-02-20T21:05:10.059734+0000","last_active":"2026-02-20T21:46:24.051709+0000","last_peered":"2026-02-20T21:46:24.051709+0000","last_clean":"2026-02-20T21:46:24.051709+0000","last_became_active":"2026-02-20T21:05:08.055463+0000","last_became_peered":"2026-02-20T21:05:08.055463+0000","last_unstale":"2026-02-20T21:46:24.051709+0000","last_undegraded":"2026-02-20T21:46:24.051709+0000","last_fullsized":"2026-02-20T21:46:24.051709+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:01:55.764306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080464600000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.738526+0000","last_change":"2026-02-20T21:05:10.570249+0000","last_active":"2026-02-20T21:46:24.738526+0000","last_peered":"2026-02-20T21:46:24.738526+0000","last_clean":"2026-02-20T21:46:24.738526+0000","last_became_active":"2026-02-20T21:05:08.055414+0000","last_became_peered":"2026-02-20T21:05:08.055414+0000","last_unstale":"2026-02-20T21:46:24.738526+0000","last_undegraded":"2026-02-20T21:46:24.738526+0000","last_fullsized":"2026-02-20T21:46:24.738526+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:02:16.632242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026386399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.738403+0000","last_change":"2026-02-20T21:05:10.570241+0000","last_active":"2026-02-20T21:46:24.738403+0000","last_peered":"2026-02-20T21:46:24.738403+0000","last_clean":"2026-02-20T21:46:24.738403+0000","last_became_active":"2026-02-20T21:05:08.055485+0000","last_became_peered":"2026-02-20T21:05:08.055485+0000","last_unstale":"2026-02-20T21:46:24.738403+0000","last_undegraded":"2026-02-20T21:46:24.738403+0000","last_fullsized":"2026-02-20T21:46:24.738403+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:51:10.827259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027411299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.610046+0000","last_change":"2026-02-20T21:05:10.057464+0000","last_active":"2026-02-20T21:46:24.610046+0000","last_peered":"2026-02-20T21:46:24.610046+0000","last_clean":"2026-02-20T21:46:24.610046+0000","last_became_active":"2026-02-20T21:05:08.055280+0000","last_became_peered":"2026-02-20T21:05:08.055280+0000","last_unstale":"2026-02-20T21:46:24.610046+0000","last_undegraded":"2026-02-20T21:46:24.610046+0000","last_fullsized":"2026-02-20T21:46:24.610046+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:37:04.239908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000404788,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":768,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.605673+0000","last_change":"2026-02-20T21:05:05.374522+0000","last_active":"2026-02-20T21:46:24.605673+0000","last_peered":"2026-02-20T21:46:24.605673+0000","last_clean":"2026-02-20T21:46:24.605673+0000","last_became_active":"2026-02-20T21:05:05.373652+0000","last_became_peered":"2026-02-20T21:05:05.373652+0000","last_unstale":"2026-02-20T21:46:24.605673+0000","last_undegraded":"2026-02-20T21:46:24.605673+0000","last_fullsized":"2026-02-20T21:46:24.605673+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:04.046826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:04.046826+0000","last_clean_scrub_stamp":"2026-02-20T21:05:04.046826+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:34:03.727741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738871,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1628304,"kb_used_data":2688,"kb_used_omap":1412,"kb_used_meta":1624187,"kb_avail":92743536,"statfs":{"total":96636764160,"available":94969380864,"internally_reserved":0,"allocated":2752512,"data_stored":1456554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1446717,"internal_metadata":1663167683},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738871,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":749528,"kb_used_data":3272,"kb_used_omap":609,"kb_used_meta":745630,"kb_avail":93622312,"statfs":{"total":96636764160,"available":95869247488,"internally_reserved":0,"allocated":3350528,"data_stored":2046941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":624184,"internal_metadata":763525576},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738873,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2003096,"kb_used_data":3272,"kb_used_omap":1116,"kb_used_meta":1998691,"kb_avail":92368744,"statfs":{"total":96636764160,"available":94585593856,"internally_reserved":0,"allocated":3350528,"data_stored":2046941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1143549,"internal_metadata":2046659843},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:46:50.741 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:46:50.927 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:46:50.928 INFO:teuthology.orchestra.run.trial069.stderr:dumped all 2026-02-20T21:46:50.937 INFO:teuthology.orchestra.run.trial069.stdout:{"pg_ready":true,"pg_map":{"version":1584,"stamp":"2026-02-20T21:46:49.427276+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4380928,"kb_used_data":9232,"kb_used_omap":3139,"kb_used_meta":4368508,"kb_avail":278734592,"statfs":{"total":289910292480,"available":285424222208,"internally_reserved":0,"allocated":9453568,"data_stored":5550436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3214450,"internal_metadata":4473353102},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000988"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.609366+0000","last_change":"2026-02-20T21:05:10.057921+0000","last_active":"2026-02-20T21:46:24.609366+0000","last_peered":"2026-02-20T21:46:24.609366+0000","last_clean":"2026-02-20T21:46:24.609366+0000","last_became_active":"2026-02-20T21:05:08.055580+0000","last_became_peered":"2026-02-20T21:05:08.055580+0000","last_unstale":"2026-02-20T21:46:24.609366+0000","last_undegraded":"2026-02-20T21:46:24.609366+0000","last_fullsized":"2026-02-20T21:46:24.609366+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:02:05.883424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079302400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.610039+0000","last_change":"2026-02-20T21:05:10.057723+0000","last_active":"2026-02-20T21:46:24.610039+0000","last_peered":"2026-02-20T21:46:24.610039+0000","last_clean":"2026-02-20T21:46:24.610039+0000","last_became_active":"2026-02-20T21:05:08.055678+0000","last_became_peered":"2026-02-20T21:05:08.055678+0000","last_unstale":"2026-02-20T21:46:24.610039+0000","last_undegraded":"2026-02-20T21:46:24.610039+0000","last_fullsized":"2026-02-20T21:46:24.610039+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:13:21.229779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061507100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.609652+0000","last_change":"2026-02-20T21:05:10.057558+0000","last_active":"2026-02-20T21:46:24.609652+0000","last_peered":"2026-02-20T21:46:24.609652+0000","last_clean":"2026-02-20T21:46:24.609652+0000","last_became_active":"2026-02-20T21:05:08.055784+0000","last_became_peered":"2026-02-20T21:05:08.055784+0000","last_unstale":"2026-02-20T21:46:24.609652+0000","last_undegraded":"2026-02-20T21:46:24.609652+0000","last_fullsized":"2026-02-20T21:46:24.609652+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:08:45.110689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046839599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.608827+0000","last_change":"2026-02-20T21:05:10.058070+0000","last_active":"2026-02-20T21:46:24.608827+0000","last_peered":"2026-02-20T21:46:24.608827+0000","last_clean":"2026-02-20T21:46:24.608827+0000","last_became_active":"2026-02-20T21:05:08.055399+0000","last_became_peered":"2026-02-20T21:05:08.055399+0000","last_unstale":"2026-02-20T21:46:24.608827+0000","last_undegraded":"2026-02-20T21:46:24.608827+0000","last_fullsized":"2026-02-20T21:46:24.608827+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:30:58.729968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091594400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":680,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.051709+0000","last_change":"2026-02-20T21:05:10.059734+0000","last_active":"2026-02-20T21:46:24.051709+0000","last_peered":"2026-02-20T21:46:24.051709+0000","last_clean":"2026-02-20T21:46:24.051709+0000","last_became_active":"2026-02-20T21:05:08.055463+0000","last_became_peered":"2026-02-20T21:05:08.055463+0000","last_unstale":"2026-02-20T21:46:24.051709+0000","last_undegraded":"2026-02-20T21:46:24.051709+0000","last_fullsized":"2026-02-20T21:46:24.051709+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:01:55.764306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080464600000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.738526+0000","last_change":"2026-02-20T21:05:10.570249+0000","last_active":"2026-02-20T21:46:24.738526+0000","last_peered":"2026-02-20T21:46:24.738526+0000","last_clean":"2026-02-20T21:46:24.738526+0000","last_became_active":"2026-02-20T21:05:08.055414+0000","last_became_peered":"2026-02-20T21:05:08.055414+0000","last_unstale":"2026-02-20T21:46:24.738526+0000","last_undegraded":"2026-02-20T21:46:24.738526+0000","last_fullsized":"2026-02-20T21:46:24.738526+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:02:16.632242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026386399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.738403+0000","last_change":"2026-02-20T21:05:10.570241+0000","last_active":"2026-02-20T21:46:24.738403+0000","last_peered":"2026-02-20T21:46:24.738403+0000","last_clean":"2026-02-20T21:46:24.738403+0000","last_became_active":"2026-02-20T21:05:08.055485+0000","last_became_peered":"2026-02-20T21:05:08.055485+0000","last_unstale":"2026-02-20T21:46:24.738403+0000","last_undegraded":"2026-02-20T21:46:24.738403+0000","last_fullsized":"2026-02-20T21:46:24.738403+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:51:10.827259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027411299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.610046+0000","last_change":"2026-02-20T21:05:10.057464+0000","last_active":"2026-02-20T21:46:24.610046+0000","last_peered":"2026-02-20T21:46:24.610046+0000","last_clean":"2026-02-20T21:46:24.610046+0000","last_became_active":"2026-02-20T21:05:08.055280+0000","last_became_peered":"2026-02-20T21:05:08.055280+0000","last_unstale":"2026-02-20T21:46:24.610046+0000","last_undegraded":"2026-02-20T21:46:24.610046+0000","last_fullsized":"2026-02-20T21:46:24.610046+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:37:04.239908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000404788,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":768,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.605673+0000","last_change":"2026-02-20T21:05:05.374522+0000","last_active":"2026-02-20T21:46:24.605673+0000","last_peered":"2026-02-20T21:46:24.605673+0000","last_clean":"2026-02-20T21:46:24.605673+0000","last_became_active":"2026-02-20T21:05:05.373652+0000","last_became_peered":"2026-02-20T21:05:05.373652+0000","last_unstale":"2026-02-20T21:46:24.605673+0000","last_undegraded":"2026-02-20T21:46:24.605673+0000","last_fullsized":"2026-02-20T21:46:24.605673+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:04.046826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:04.046826+0000","last_clean_scrub_stamp":"2026-02-20T21:05:04.046826+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:34:03.727741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738871,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1628304,"kb_used_data":2688,"kb_used_omap":1412,"kb_used_meta":1624187,"kb_avail":92743536,"statfs":{"total":96636764160,"available":94969380864,"internally_reserved":0,"allocated":2752512,"data_stored":1456554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1446717,"internal_metadata":1663167683},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738871,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":749528,"kb_used_data":3272,"kb_used_omap":609,"kb_used_meta":745630,"kb_avail":93622312,"statfs":{"total":96636764160,"available":95869247488,"internally_reserved":0,"allocated":3350528,"data_stored":2046941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":624184,"internal_metadata":763525576},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738873,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2003096,"kb_used_data":3272,"kb_used_omap":1116,"kb_used_meta":1998691,"kb_avail":92368744,"statfs":{"total":96636764160,"available":94585593856,"internally_reserved":0,"allocated":3350528,"data_stored":2046941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1143549,"internal_metadata":2046659843},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:46:50.939 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-20T21:46:50.940 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:46:51.125 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:46:51.126 INFO:teuthology.orchestra.run.trial069.stderr:dumped all 2026-02-20T21:46:51.135 INFO:teuthology.orchestra.run.trial069.stdout:{"pg_ready":true,"pg_map":{"version":1584,"stamp":"2026-02-20T21:46:49.427276+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4380928,"kb_used_data":9232,"kb_used_omap":3139,"kb_used_meta":4368508,"kb_avail":278734592,"statfs":{"total":289910292480,"available":285424222208,"internally_reserved":0,"allocated":9453568,"data_stored":5550436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3214450,"internal_metadata":4473353102},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000988"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.609366+0000","last_change":"2026-02-20T21:05:10.057921+0000","last_active":"2026-02-20T21:46:24.609366+0000","last_peered":"2026-02-20T21:46:24.609366+0000","last_clean":"2026-02-20T21:46:24.609366+0000","last_became_active":"2026-02-20T21:05:08.055580+0000","last_became_peered":"2026-02-20T21:05:08.055580+0000","last_unstale":"2026-02-20T21:46:24.609366+0000","last_undegraded":"2026-02-20T21:46:24.609366+0000","last_fullsized":"2026-02-20T21:46:24.609366+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:02:05.883424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079302400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.610039+0000","last_change":"2026-02-20T21:05:10.057723+0000","last_active":"2026-02-20T21:46:24.610039+0000","last_peered":"2026-02-20T21:46:24.610039+0000","last_clean":"2026-02-20T21:46:24.610039+0000","last_became_active":"2026-02-20T21:05:08.055678+0000","last_became_peered":"2026-02-20T21:05:08.055678+0000","last_unstale":"2026-02-20T21:46:24.610039+0000","last_undegraded":"2026-02-20T21:46:24.610039+0000","last_fullsized":"2026-02-20T21:46:24.610039+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:13:21.229779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061507100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.609652+0000","last_change":"2026-02-20T21:05:10.057558+0000","last_active":"2026-02-20T21:46:24.609652+0000","last_peered":"2026-02-20T21:46:24.609652+0000","last_clean":"2026-02-20T21:46:24.609652+0000","last_became_active":"2026-02-20T21:05:08.055784+0000","last_became_peered":"2026-02-20T21:05:08.055784+0000","last_unstale":"2026-02-20T21:46:24.609652+0000","last_undegraded":"2026-02-20T21:46:24.609652+0000","last_fullsized":"2026-02-20T21:46:24.609652+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:08:45.110689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046839599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.608827+0000","last_change":"2026-02-20T21:05:10.058070+0000","last_active":"2026-02-20T21:46:24.608827+0000","last_peered":"2026-02-20T21:46:24.608827+0000","last_clean":"2026-02-20T21:46:24.608827+0000","last_became_active":"2026-02-20T21:05:08.055399+0000","last_became_peered":"2026-02-20T21:05:08.055399+0000","last_unstale":"2026-02-20T21:46:24.608827+0000","last_undegraded":"2026-02-20T21:46:24.608827+0000","last_fullsized":"2026-02-20T21:46:24.608827+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:30:58.729968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091594400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":680,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.051709+0000","last_change":"2026-02-20T21:05:10.059734+0000","last_active":"2026-02-20T21:46:24.051709+0000","last_peered":"2026-02-20T21:46:24.051709+0000","last_clean":"2026-02-20T21:46:24.051709+0000","last_became_active":"2026-02-20T21:05:08.055463+0000","last_became_peered":"2026-02-20T21:05:08.055463+0000","last_unstale":"2026-02-20T21:46:24.051709+0000","last_undegraded":"2026-02-20T21:46:24.051709+0000","last_fullsized":"2026-02-20T21:46:24.051709+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:01:55.764306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080464600000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.738526+0000","last_change":"2026-02-20T21:05:10.570249+0000","last_active":"2026-02-20T21:46:24.738526+0000","last_peered":"2026-02-20T21:46:24.738526+0000","last_clean":"2026-02-20T21:46:24.738526+0000","last_became_active":"2026-02-20T21:05:08.055414+0000","last_became_peered":"2026-02-20T21:05:08.055414+0000","last_unstale":"2026-02-20T21:46:24.738526+0000","last_undegraded":"2026-02-20T21:46:24.738526+0000","last_fullsized":"2026-02-20T21:46:24.738526+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:02:16.632242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026386399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.738403+0000","last_change":"2026-02-20T21:05:10.570241+0000","last_active":"2026-02-20T21:46:24.738403+0000","last_peered":"2026-02-20T21:46:24.738403+0000","last_clean":"2026-02-20T21:46:24.738403+0000","last_became_active":"2026-02-20T21:05:08.055485+0000","last_became_peered":"2026-02-20T21:05:08.055485+0000","last_unstale":"2026-02-20T21:46:24.738403+0000","last_undegraded":"2026-02-20T21:46:24.738403+0000","last_fullsized":"2026-02-20T21:46:24.738403+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:51:10.827259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027411299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.610046+0000","last_change":"2026-02-20T21:05:10.057464+0000","last_active":"2026-02-20T21:46:24.610046+0000","last_peered":"2026-02-20T21:46:24.610046+0000","last_clean":"2026-02-20T21:46:24.610046+0000","last_became_active":"2026-02-20T21:05:08.055280+0000","last_became_peered":"2026-02-20T21:05:08.055280+0000","last_unstale":"2026-02-20T21:46:24.610046+0000","last_undegraded":"2026-02-20T21:46:24.610046+0000","last_fullsized":"2026-02-20T21:46:24.610046+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:37:04.239908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000404788,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":768,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.605673+0000","last_change":"2026-02-20T21:05:05.374522+0000","last_active":"2026-02-20T21:46:24.605673+0000","last_peered":"2026-02-20T21:46:24.605673+0000","last_clean":"2026-02-20T21:46:24.605673+0000","last_became_active":"2026-02-20T21:05:05.373652+0000","last_became_peered":"2026-02-20T21:05:05.373652+0000","last_unstale":"2026-02-20T21:46:24.605673+0000","last_undegraded":"2026-02-20T21:46:24.605673+0000","last_fullsized":"2026-02-20T21:46:24.605673+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:04.046826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:04.046826+0000","last_clean_scrub_stamp":"2026-02-20T21:05:04.046826+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:34:03.727741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738871,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1628304,"kb_used_data":2688,"kb_used_omap":1412,"kb_used_meta":1624187,"kb_avail":92743536,"statfs":{"total":96636764160,"available":94969380864,"internally_reserved":0,"allocated":2752512,"data_stored":1456554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1446717,"internal_metadata":1663167683},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738871,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":749528,"kb_used_data":3272,"kb_used_omap":609,"kb_used_meta":745630,"kb_avail":93622312,"statfs":{"total":96636764160,"available":95869247488,"internally_reserved":0,"allocated":3350528,"data_stored":2046941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":624184,"internal_metadata":763525576},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738873,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2003096,"kb_used_data":3272,"kb_used_omap":1116,"kb_used_meta":1998691,"kb_avail":92368744,"statfs":{"total":96636764160,"available":94585593856,"internally_reserved":0,"allocated":3350528,"data_stored":2046941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1143549,"internal_metadata":2046659843},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:46:51.137 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-20T21:46:51.321 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:46:51.322 INFO:teuthology.orchestra.run.trial069.stdout:{"epoch":334,"fsid":"4aabec94-a67f-4d24-9557-469e7f74026e","created":"2026-02-20T21:04:58.813036+0000","modified":"2026-02-20T21:46:24.044435+0000","last_up_change":"2026-02-20T21:05:02.821362+0000","last_in_change":"2026-02-20T21:04:59.484260+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"tentacle","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-20T21:05:03.055498+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-20T21:05:06.677117+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"nonprimary_shards":"{}","options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"567f2427-666c-49f4-aff5-54e91a2350f8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6800","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6801","nonce":67491597}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6802","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6803","nonce":67491597}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6806","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6807","nonce":67491597}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6804","nonce":67491597},{"type":"v1","addr":"10.20.193.69:6805","nonce":67491597}]},"public_addr":"10.20.193.69:6801/67491597","cluster_addr":"10.20.193.69:6803/67491597","heartbeat_back_addr":"10.20.193.69:6807/67491597","heartbeat_front_addr":"10.20.193.69:6805/67491597","state":["exists","up"]},{"osd":1,"uuid":"c06ca60d-cc95-4882-95d9-c4751ba99875","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":29,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6808","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6809","nonce":4090895406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6810","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6811","nonce":4090895406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6814","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6815","nonce":4090895406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6812","nonce":4090895406},{"type":"v1","addr":"10.20.193.69:6813","nonce":4090895406}]},"public_addr":"10.20.193.69:6809/4090895406","cluster_addr":"10.20.193.69:6811/4090895406","heartbeat_back_addr":"10.20.193.69:6815/4090895406","heartbeat_front_addr":"10.20.193.69:6813/4090895406","state":["exists","up"]},{"osd":2,"uuid":"6805f378-c47b-4987-8d01-3c839ec7d272","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":32,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6816","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6817","nonce":1475133124}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6818","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6819","nonce":1475133124}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6822","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6823","nonce":1475133124}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.69:6820","nonce":1475133124},{"type":"v1","addr":"10.20.193.69:6821","nonce":1475133124}]},"public_addr":"10.20.193.69:6817/1475133124","cluster_addr":"10.20.193.69:6819/1475133124","heartbeat_back_addr":"10.20.193.69:6823/1475133124","heartbeat_front_addr":"10.20.193.69:6821/1475133124","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:01.438954+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:01.364795+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4544132024016699391,"old_weight":0,"last_purged_snaps_scrub":"2026-02-20T21:05:01.445158+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"isa","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-20T21:46:52.332 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-20T21:46:52.332 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-20T21:46:52.416 INFO:teuthology.orchestra.run.trial069.stdout:{ 2026-02-20T21:46:52.416 INFO:teuthology.orchestra.run.trial069.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T21:46:52.416 INFO:teuthology.orchestra.run.trial069.stdout:} 2026-02-20T21:46:52.423 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-20T21:46:52.605 INFO:teuthology.orchestra.run.trial069.stderr:instructed osd(s) 0 to deep-scrub 2026-02-20T21:46:52.614 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-20T21:46:52.614 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-20T21:46:52.697 INFO:teuthology.orchestra.run.trial069.stdout:{ 2026-02-20T21:46:52.697 INFO:teuthology.orchestra.run.trial069.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-20T21:46:52.697 INFO:teuthology.orchestra.run.trial069.stdout:} 2026-02-20T21:46:52.703 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-20T21:46:52.886 INFO:teuthology.orchestra.run.trial069.stderr:instructed osd(s) 1 to deep-scrub 2026-02-20T21:46:52.895 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-20T21:46:52.895 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-20T21:46:52.979 INFO:teuthology.orchestra.run.trial069.stdout:{ 2026-02-20T21:46:52.979 INFO:teuthology.orchestra.run.trial069.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-20T21:46:52.979 INFO:teuthology.orchestra.run.trial069.stdout:} 2026-02-20T21:46:52.986 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-20T21:46:53.166 INFO:teuthology.orchestra.run.trial069.stderr:instructed osd(s) 2 to deep-scrub 2026-02-20T21:46:53.175 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:46:53.357 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:46:53.357 INFO:teuthology.orchestra.run.trial069.stderr:dumped all 2026-02-20T21:46:53.365 INFO:teuthology.orchestra.run.trial069.stdout:{"pg_ready":true,"pg_map":{"version":1585,"stamp":"2026-02-20T21:46:51.427464+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4380928,"kb_used_data":9232,"kb_used_omap":3139,"kb_used_meta":4368508,"kb_avail":278734592,"statfs":{"total":289910292480,"available":285424222208,"internally_reserved":0,"allocated":9453568,"data_stored":5550436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3214450,"internal_metadata":4473353102},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001007"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.609366+0000","last_change":"2026-02-20T21:05:10.057921+0000","last_active":"2026-02-20T21:46:24.609366+0000","last_peered":"2026-02-20T21:46:24.609366+0000","last_clean":"2026-02-20T21:46:24.609366+0000","last_became_active":"2026-02-20T21:05:08.055580+0000","last_became_peered":"2026-02-20T21:05:08.055580+0000","last_unstale":"2026-02-20T21:46:24.609366+0000","last_undegraded":"2026-02-20T21:46:24.609366+0000","last_fullsized":"2026-02-20T21:46:24.609366+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:02:05.883424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00079302400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.610039+0000","last_change":"2026-02-20T21:05:10.057723+0000","last_active":"2026-02-20T21:46:24.610039+0000","last_peered":"2026-02-20T21:46:24.610039+0000","last_clean":"2026-02-20T21:46:24.610039+0000","last_became_active":"2026-02-20T21:05:08.055678+0000","last_became_peered":"2026-02-20T21:05:08.055678+0000","last_unstale":"2026-02-20T21:46:24.610039+0000","last_undegraded":"2026-02-20T21:46:24.610039+0000","last_fullsized":"2026-02-20T21:46:24.610039+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T22:13:21.229779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061507100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":677,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.609652+0000","last_change":"2026-02-20T21:05:10.057558+0000","last_active":"2026-02-20T21:46:24.609652+0000","last_peered":"2026-02-20T21:46:24.609652+0000","last_clean":"2026-02-20T21:46:24.609652+0000","last_became_active":"2026-02-20T21:05:08.055784+0000","last_became_peered":"2026-02-20T21:05:08.055784+0000","last_unstale":"2026-02-20T21:46:24.609652+0000","last_undegraded":"2026-02-20T21:46:24.609652+0000","last_fullsized":"2026-02-20T21:46:24.609652+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:08:45.110689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046839599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.608827+0000","last_change":"2026-02-20T21:05:10.058070+0000","last_active":"2026-02-20T21:46:24.608827+0000","last_peered":"2026-02-20T21:46:24.608827+0000","last_clean":"2026-02-20T21:46:24.608827+0000","last_became_active":"2026-02-20T21:05:08.055399+0000","last_became_peered":"2026-02-20T21:05:08.055399+0000","last_unstale":"2026-02-20T21:46:24.608827+0000","last_undegraded":"2026-02-20T21:46:24.608827+0000","last_fullsized":"2026-02-20T21:46:24.608827+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:30:58.729968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091594400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":680,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.051709+0000","last_change":"2026-02-20T21:05:10.059734+0000","last_active":"2026-02-20T21:46:24.051709+0000","last_peered":"2026-02-20T21:46:24.051709+0000","last_clean":"2026-02-20T21:46:24.051709+0000","last_became_active":"2026-02-20T21:05:08.055463+0000","last_became_peered":"2026-02-20T21:05:08.055463+0000","last_unstale":"2026-02-20T21:46:24.051709+0000","last_undegraded":"2026-02-20T21:46:24.051709+0000","last_fullsized":"2026-02-20T21:46:24.051709+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:01:55.764306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080464600000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.738526+0000","last_change":"2026-02-20T21:05:10.570249+0000","last_active":"2026-02-20T21:46:24.738526+0000","last_peered":"2026-02-20T21:46:24.738526+0000","last_clean":"2026-02-20T21:46:24.738526+0000","last_became_active":"2026-02-20T21:05:08.055414+0000","last_became_peered":"2026-02-20T21:05:08.055414+0000","last_unstale":"2026-02-20T21:46:24.738526+0000","last_undegraded":"2026-02-20T21:46:24.738526+0000","last_fullsized":"2026-02-20T21:46:24.738526+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T01:02:16.632242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026386399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":678,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.738403+0000","last_change":"2026-02-20T21:05:10.570241+0000","last_active":"2026-02-20T21:46:24.738403+0000","last_peered":"2026-02-20T21:46:24.738403+0000","last_clean":"2026-02-20T21:46:24.738403+0000","last_became_active":"2026-02-20T21:05:08.055485+0000","last_became_peered":"2026-02-20T21:05:08.055485+0000","last_unstale":"2026-02-20T21:46:24.738403+0000","last_undegraded":"2026-02-20T21:46:24.738403+0000","last_fullsized":"2026-02-20T21:46:24.738403+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-21T21:51:10.827259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027411299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":679,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.610046+0000","last_change":"2026-02-20T21:05:10.057464+0000","last_active":"2026-02-20T21:46:24.610046+0000","last_peered":"2026-02-20T21:46:24.610046+0000","last_clean":"2026-02-20T21:46:24.610046+0000","last_became_active":"2026-02-20T21:05:08.055280+0000","last_became_peered":"2026-02-20T21:05:08.055280+0000","last_unstale":"2026-02-20T21:46:24.610046+0000","last_undegraded":"2026-02-20T21:46:24.610046+0000","last_fullsized":"2026-02-20T21:46:24.610046+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:07.049872+0000","last_clean_scrub_stamp":"2026-02-20T21:05:07.049872+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:37:04.239908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000404788,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"10'72","reported_seq":768,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:24.605673+0000","last_change":"2026-02-20T21:05:05.374522+0000","last_active":"2026-02-20T21:46:24.605673+0000","last_peered":"2026-02-20T21:46:24.605673+0000","last_clean":"2026-02-20T21:46:24.605673+0000","last_became_active":"2026-02-20T21:05:05.373652+0000","last_became_peered":"2026-02-20T21:05:05.373652+0000","last_unstale":"2026-02-20T21:46:24.605673+0000","last_undegraded":"2026-02-20T21:46:24.605673+0000","last_fullsized":"2026-02-20T21:46:24.605673+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:05:04.046826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:05:04.046826+0000","last_clean_scrub_stamp":"2026-02-20T21:05:04.046826+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:34:03.727741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738871,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1628304,"kb_used_data":2688,"kb_used_omap":1412,"kb_used_meta":1624187,"kb_avail":92743536,"statfs":{"total":96636764160,"available":94969380864,"internally_reserved":0,"allocated":2752512,"data_stored":1456554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1446717,"internal_metadata":1663167683},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738871,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":749528,"kb_used_data":3272,"kb_used_omap":609,"kb_used_meta":745630,"kb_avail":93622312,"statfs":{"total":96636764160,"available":95869247488,"internally_reserved":0,"allocated":3350528,"data_stored":2046941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":624184,"internal_metadata":763525576},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738873,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2003096,"kb_used_data":3272,"kb_used_omap":1116,"kb_used_meta":1998691,"kb_avail":92368744,"statfs":{"total":96636764160,"available":94585593856,"internally_reserved":0,"allocated":3350528,"data_stored":2046941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1143549,"internal_metadata":2046659843},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:46:53.366 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-20T21:05:07.049872+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:53.366 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-20T21:05:07.049872+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:53.366 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-20T21:05:07.049872+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:53.366 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-20T21:05:07.049872+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:53.366 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-20T21:05:07.049872+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:53.366 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-20T21:05:07.049872+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:53.366 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-20T21:05:07.049872+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:53.366 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-20T21:05:07.049872+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=7, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:53.366 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-20T21:05:04.046826+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=5, tm_sec=4, tm_wday=4, tm_yday=51, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=20, tm_hour=21, tm_min=46, tm_sec=51, tm_wday=4, tm_yday=51, tm_isdst=0) 2026-02-20T21:46:53.366 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-20T21:47:13.367 DEBUG:teuthology.orchestra.run.trial069:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-20T21:47:13.551 INFO:teuthology.orchestra.run.trial069.stdout: 2026-02-20T21:47:13.552 INFO:teuthology.orchestra.run.trial069.stderr:dumped all 2026-02-20T21:47:13.561 INFO:teuthology.orchestra.run.trial069.stdout:{"pg_ready":true,"pg_map":{"version":1596,"stamp":"2026-02-20T21:47:13.429519+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":4380928,"kb_used_data":9232,"kb_used_omap":3139,"kb_used_meta":4368508,"kb_avail":278734592,"statfs":{"total":289910292480,"available":285424222208,"internally_reserved":0,"allocated":9453568,"data_stored":5550436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3214450,"internal_metadata":4473353102},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000936"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":685,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:58.599727+0000","last_change":"2026-02-20T21:46:58.599727+0000","last_active":"2026-02-20T21:46:58.599727+0000","last_peered":"2026-02-20T21:46:58.599727+0000","last_clean":"2026-02-20T21:46:58.599727+0000","last_became_active":"2026-02-20T21:05:08.055580+0000","last_became_peered":"2026-02-20T21:05:08.055580+0000","last_unstale":"2026-02-20T21:46:58.599727+0000","last_undegraded":"2026-02-20T21:46:58.599727+0000","last_fullsized":"2026-02-20T21:46:58.599727+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:46:58.599676+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:46:58.599676+0000","last_clean_scrub_stamp":"2026-02-20T21:46:58.599676+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T04:31:00.109922+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0.00079302400000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":685,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:57.552246+0000","last_change":"2026-02-20T21:46:57.552246+0000","last_active":"2026-02-20T21:46:57.552246+0000","last_peered":"2026-02-20T21:46:57.552246+0000","last_clean":"2026-02-20T21:46:57.552246+0000","last_became_active":"2026-02-20T21:05:08.055678+0000","last_became_peered":"2026-02-20T21:05:08.055678+0000","last_unstale":"2026-02-20T21:46:57.552246+0000","last_undegraded":"2026-02-20T21:46:57.552246+0000","last_fullsized":"2026-02-20T21:46:57.552246+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:46:57.552170+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:46:57.552170+0000","last_clean_scrub_stamp":"2026-02-20T21:46:57.552170+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:28:34.476908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061507100000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":686,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:55.563012+0000","last_change":"2026-02-20T21:46:55.562963+0000","last_active":"2026-02-20T21:46:55.563012+0000","last_peered":"2026-02-20T21:46:55.563012+0000","last_clean":"2026-02-20T21:46:55.563012+0000","last_became_active":"2026-02-20T21:05:08.055784+0000","last_became_peered":"2026-02-20T21:05:08.055784+0000","last_unstale":"2026-02-20T21:46:55.563012+0000","last_undegraded":"2026-02-20T21:46:55.563012+0000","last_fullsized":"2026-02-20T21:46:55.563012+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:46:55.562926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:46:55.562926+0000","last_clean_scrub_stamp":"2026-02-20T21:46:55.562926+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T06:13:21.504764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046839599999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":686,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:54.563232+0000","last_change":"2026-02-20T21:46:54.563232+0000","last_active":"2026-02-20T21:46:54.563232+0000","last_peered":"2026-02-20T21:46:54.563232+0000","last_clean":"2026-02-20T21:46:54.563232+0000","last_became_active":"2026-02-20T21:05:08.055399+0000","last_became_peered":"2026-02-20T21:05:08.055399+0000","last_unstale":"2026-02-20T21:46:54.563232+0000","last_undegraded":"2026-02-20T21:46:54.563232+0000","last_fullsized":"2026-02-20T21:46:54.563232+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:46:54.563178+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:46:54.563178+0000","last_clean_scrub_stamp":"2026-02-20T21:46:54.563178+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T09:16:05.268159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00091594400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"334'3","reported_seq":689,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:52.746488+0000","last_change":"2026-02-20T21:46:52.746488+0000","last_active":"2026-02-20T21:46:52.746488+0000","last_peered":"2026-02-20T21:46:52.746488+0000","last_clean":"2026-02-20T21:46:52.746488+0000","last_became_active":"2026-02-20T21:05:08.055463+0000","last_became_peered":"2026-02-20T21:05:08.055463+0000","last_unstale":"2026-02-20T21:46:52.746488+0000","last_undegraded":"2026-02-20T21:46:52.746488+0000","last_fullsized":"2026-02-20T21:46:52.746488+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"334'3","last_scrub_stamp":"2026-02-20T21:46:52.746444+0000","last_deep_scrub":"334'3","last_deep_scrub_stamp":"2026-02-20T21:46:52.746444+0000","last_clean_scrub_stamp":"2026-02-20T21:46:52.746444+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T00:22:26.465647+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.00080464600000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":686,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:54.892856+0000","last_change":"2026-02-20T21:46:54.892856+0000","last_active":"2026-02-20T21:46:54.892856+0000","last_peered":"2026-02-20T21:46:54.892856+0000","last_clean":"2026-02-20T21:46:54.892856+0000","last_became_active":"2026-02-20T21:05:08.055414+0000","last_became_peered":"2026-02-20T21:05:08.055414+0000","last_unstale":"2026-02-20T21:46:54.892856+0000","last_undegraded":"2026-02-20T21:46:54.892856+0000","last_fullsized":"2026-02-20T21:46:54.892856+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:46:54.892814+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:46:54.892814+0000","last_clean_scrub_stamp":"2026-02-20T21:46:54.892814+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T05:15:40.986850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026386399999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":686,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:53.888555+0000","last_change":"2026-02-20T21:46:53.888555+0000","last_active":"2026-02-20T21:46:53.888555+0000","last_peered":"2026-02-20T21:46:53.888555+0000","last_clean":"2026-02-20T21:46:53.888555+0000","last_became_active":"2026-02-20T21:05:08.055485+0000","last_became_peered":"2026-02-20T21:05:08.055485+0000","last_unstale":"2026-02-20T21:46:53.888555+0000","last_undegraded":"2026-02-20T21:46:53.888555+0000","last_fullsized":"2026-02-20T21:46:53.888555+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-20T21:46:53.888519+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-20T21:46:53.888519+0000","last_clean_scrub_stamp":"2026-02-20T21:46:53.888519+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T03:55:52.906607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027411299999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"334'2","reported_seq":688,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:56.594725+0000","last_change":"2026-02-20T21:46:56.594725+0000","last_active":"2026-02-20T21:46:56.594725+0000","last_peered":"2026-02-20T21:46:56.594725+0000","last_clean":"2026-02-20T21:46:56.594725+0000","last_became_active":"2026-02-20T21:05:08.055280+0000","last_became_peered":"2026-02-20T21:05:08.055280+0000","last_unstale":"2026-02-20T21:46:56.594725+0000","last_undegraded":"2026-02-20T21:46:56.594725+0000","last_fullsized":"2026-02-20T21:46:56.594725+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"334'2","last_scrub_stamp":"2026-02-20T21:46:56.594645+0000","last_deep_scrub":"334'2","last_deep_scrub_stamp":"2026-02-20T21:46:56.594645+0000","last_clean_scrub_stamp":"2026-02-20T21:46:56.594645+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T08:30:59.833340+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0.000404788,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"334'74","reported_seq":778,"reported_epoch":334,"state":"active+clean","last_fresh":"2026-02-20T21:46:53.518752+0000","last_change":"2026-02-20T21:46:53.518752+0000","last_active":"2026-02-20T21:46:53.518752+0000","last_peered":"2026-02-20T21:46:53.518752+0000","last_clean":"2026-02-20T21:46:53.518752+0000","last_became_active":"2026-02-20T21:05:05.373652+0000","last_became_peered":"2026-02-20T21:05:05.373652+0000","last_unstale":"2026-02-20T21:46:53.518752+0000","last_undegraded":"2026-02-20T21:46:53.518752+0000","last_fullsized":"2026-02-20T21:46:53.518752+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"334'74","last_scrub_stamp":"2026-02-20T21:46:53.518710+0000","last_deep_scrub":"334'74","last_deep_scrub_stamp":"2026-02-20T21:46:53.518710+0000","last_clean_scrub_stamp":"2026-02-20T21:46:53.518710+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-22T02:00:39.636070+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738876,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":1628304,"kb_used_data":2688,"kb_used_omap":1412,"kb_used_meta":1624187,"kb_avail":92743536,"statfs":{"total":96636764160,"available":94969380864,"internally_reserved":0,"allocated":2752512,"data_stored":1456554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1446717,"internal_metadata":1663167683},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738876,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":749528,"kb_used_data":3272,"kb_used_omap":609,"kb_used_meta":745630,"kb_avail":93622312,"statfs":{"total":96636764160,"available":95869247488,"internally_reserved":0,"allocated":3350528,"data_stored":2046941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":624184,"internal_metadata":763525576},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738878,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":2003096,"kb_used_data":3272,"kb_used_omap":1116,"kb_used_meta":1998691,"kb_avail":92368744,"statfs":{"total":96636764160,"available":94585593856,"internally_reserved":0,"allocated":3350528,"data_stored":2046941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1143549,"internal_metadata":2046659843},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-20T21:47:13.565 DEBUG:teuthology.orchestra.run.trial069:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-20T21:47:13.761 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-20T21:47:13.761 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-20T21:47:13.761 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-20T21:47:13.761 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:47:13.816 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-20T21:47:13.816 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-20T21:47:13.817 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:47:13.890 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-20T21:47:13.891 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-20T21:47:13.891 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:47:13.951 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-20T21:47:13.951 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-20T21:47:13.951 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-20T21:47:13.952 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:47:13.980 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-20T21:47:13.980 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-20T21:47:13.980 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-20T21:47:13.980 INFO:teuthology.orchestra.run:waiting for 300 2026-02-20T21:47:14.033 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-20T21:47:14.033 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-20T21:47:14.034 DEBUG:teuthology.orchestra.run.trial069:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-20T21:47:14.100 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial069.front.sepia.ceph.com 2026-02-20T21:47:14.100 DEBUG:teuthology.orchestra.run.trial069:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-20T21:47:14.322 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial069.front.sepia.ceph.com 2026-02-20T21:47:14.322 DEBUG:teuthology.orchestra.run.trial069:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-20T21:47:14.472 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial069.front.sepia.ceph.com 2026-02-20T21:47:14.472 DEBUG:teuthology.orchestra.run.trial069:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-20T21:47:14.631 INFO:tasks.ceph:Archiving mon data... 2026-02-20T21:47:14.632 DEBUG:teuthology.misc:Transferring archived files from trial069:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424/data/cluster1.mon.a.tgz 2026-02-20T21:47:14.632 DEBUG:teuthology.orchestra.run.trial069:> mktemp 2026-02-20T21:47:14.649 INFO:teuthology.orchestra.run.trial069.stdout:/tmp/tmp.dUnFxtXBWV 2026-02-20T21:47:14.649 DEBUG:teuthology.orchestra.run.trial069:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.dUnFxtXBWV 2026-02-20T21:47:14.758 DEBUG:teuthology.orchestra.run.trial069:> sudo chmod 0666 /tmp/tmp.dUnFxtXBWV 2026-02-20T21:47:14.841 DEBUG:teuthology.orchestra.remote:trial069:/tmp/tmp.dUnFxtXBWV is 348KB 2026-02-20T21:47:14.906 DEBUG:teuthology.orchestra.run.trial069:> rm -fr /tmp/tmp.dUnFxtXBWV 2026-02-20T21:47:14.923 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-20T21:47:14.923 DEBUG:teuthology.orchestra.run.trial069:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-20T21:47:14.965 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-20T21:47:15.043 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-20T21:47:15.061 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-20T21:47:15.061 INFO:tasks.ceph:Archiving crash dumps... 2026-02-20T21:47:15.063 DEBUG:teuthology.misc:Transferring archived files from trial069:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424/remote/trial069/crash 2026-02-20T21:47:15.064 DEBUG:teuthology.orchestra.run.trial069:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-20T21:47:15.109 DEBUG:teuthology.misc:Transferring archived files from trial163:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424/remote/trial163/crash 2026-02-20T21:47:15.110 DEBUG:teuthology.orchestra.run.trial163:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-20T21:47:15.135 INFO:tasks.ceph:Compressing logs... 2026-02-20T21:47:15.136 DEBUG:teuthology.orchestra.run.trial069:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-20T21:47:15.154 DEBUG:teuthology.orchestra.run.trial163:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-20T21:47:15.189 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46742.log 2026-02-20T21:47:15.189 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46840.log 2026-02-20T21:47:15.190 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.46742.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46742.log.gz 2026-02-20T21:47:15.190 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41780.log 2026-02-20T21:47:15.190 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.46840.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42104.log 2026-02-20T21:47:15.190 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46840.log.gz 2026-02-20T21:47:15.190 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.41780.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-20T21:47:15.190 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41780.log.gz 2026-02-20T21:47:15.190 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.42104.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-20T21:47:15.190 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42104.log.gz 2026-02-20T21:47:15.190 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-02-20T21:47:15.190 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45150.log 2026-02-20T21:47:15.190 INFO:teuthology.orchestra.run.trial069.stderr:gzip/var/log/ceph/cluster1-osd.2.log: -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-20T21:47:15.191 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45150.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.36879.log 2026-02-20T21:47:15.191 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45150.log.gz 2026-02-20T21:47:15.191 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45738.log 2026-02-20T21:47:15.191 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45085.log 2026-02-20T21:47:15.191 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/ceph.tmp-client.admin.36879.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.36879.log.gz 2026-02-20T21:47:15.191 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45738.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46497.log 2026-02-20T21:47:15.191 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45738.log.gz 2026-02-20T21:47:15.191 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45085.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-20T21:47:15.191 INFO:teuthology.orchestra.run.trial069.stderr: 80.4% -- replaced with /var/log/ceph/cluster1-client.admin.45085.log.gz 2026-02-20T21:47:15.191 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.46497.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45399.log 2026-02-20T21:47:15.191 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46497.log.gz 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46448.log 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45399.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46546.log 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45399.log.gz 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.46448.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42010.log 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46448.log.gz 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.46546.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45248.log 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46546.log.gz 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.42010.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.44990.log 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42010.log.gz 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45248.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46791.log 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45248.log.gz 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.44990.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46889.log 2026-02-20T21:47:15.192 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.44990.log.gz 2026-02-20T21:47:15.193 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.46791.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-02-20T21:47:15.193 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46791.log.gz 2026-02-20T21:47:15.193 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.46889.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45297.log 2026-02-20T21:47:15.193 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46889.log.gz 2026-02-20T21:47:15.193 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46987.log 2026-02-20T21:47:15.193 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45297.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46693.log 2026-02-20T21:47:15.193 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45297.log.gz 2026-02-20T21:47:15.193 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.46987.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46987.log.gz 2026-02-20T21:47:15.193 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42374.log 2026-02-20T21:47:15.193 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.46693.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45199.log 2026-02-20T21:47:15.193 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46693.log.gz 2026-02-20T21:47:15.193 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.42374.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42374.log.gz 2026-02-20T21:47:15.194 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45547.log 2026-02-20T21:47:15.194 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45199.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45546.log 2026-02-20T21:47:15.194 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45199.log.gz 2026-02-20T21:47:15.194 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45547.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45547.log.gz 2026-02-20T21:47:15.194 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47034.log 2026-02-20T21:47:15.194 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45546.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45640.log 2026-02-20T21:47:15.194 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45546.log.gz 2026-02-20T21:47:15.194 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.47034.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47034.log.gz 2026-02-20T21:47:15.194 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45060.log 2026-02-20T21:47:15.194 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45640.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45689.log 2026-02-20T21:47:15.194 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45640.log.gz 2026-02-20T21:47:15.194 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45060.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45060.log.gz 2026-02-20T21:47:15.195 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-20T21:47:15.195 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45689.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45545.log 2026-02-20T21:47:15.195 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45689.log.gz 2026-02-20T21:47:15.195 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46938.log 2026-02-20T21:47:15.195 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45545.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42231.log 2026-02-20T21:47:15.195 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45545.log.gz 2026-02-20T21:47:15.195 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.46938.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46938.log.gz 2026-02-20T21:47:15.195 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46595.log 2026-02-20T21:47:15.195 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.42231.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.41876.log 2026-02-20T21:47:15.195 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42231.log.gz 2026-02-20T21:47:15.195 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.46595.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46595.log.gz 2026-02-20T21:47:15.195 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45400.log 2026-02-20T21:47:15.196 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.41876.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.41876.log.gz 2026-02-20T21:47:15.196 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45398.log 2026-02-20T21:47:15.196 INFO:teuthology.orchestra.run.trial069.stderr: 93.2% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-20T21:47:15.196 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45400.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45400.log.gz 2026-02-20T21:47:15.196 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-02-20T21:47:15.196 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.45398.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45398.log.gz 2026-02-20T21:47:15.196 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46644.log 2026-02-20T21:47:15.196 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42057.log 2026-02-20T21:47:15.196 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.46644.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46644.log.gz 2026-02-20T21:47:15.196 INFO:teuthology.orchestra.run.trial069.stderr:/var/log/ceph/cluster1-client.admin.42057.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42057.log.gz 2026-02-20T21:47:15.196 INFO:teuthology.orchestra.run.trial069.stderr: 88.1% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-20T21:47:15.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97147.log 2026-02-20T21:47:15.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137239.log 2026-02-20T21:47:15.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip/var/log/ceph/cluster1-client.mirror.97147.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108110.log 2026-02-20T21:47:15.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137239.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141075.log 2026-02-20T21:47:15.242 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137239.log.gz 2026-02-20T21:47:15.242 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.97147.log.gz 2026-02-20T21:47:15.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108110.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78868.log 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141075.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108110.log.gz 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126346.log 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.141075.log.gz 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78868.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129714.log 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.78868.log.gz 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111099.log 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.126346.log: /var/log/ceph/cluster2-client.mirror.129714.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102521.log 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.129714.log.gz 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111099.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71163.log 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.111099.log.gz 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102521.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54208.log 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.126346.log.gz 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.102521.log.gz 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71163.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91908.log 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.54208.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71163.log.gz 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141882.log 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.54208.log.gz 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91908.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136303.log 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141882.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141776.log 2026-02-20T21:47:15.243 INFO:teuthology.orchestra.run.trial163.stderr: 83.4% -- replaced with /var/log/ceph/cluster2-client.mirror.141882.log.gz 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79612.log 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136303.log: /var/log/ceph/cluster1-client.mirror.141776.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47935.log 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.136303.log.gz 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.141776.log.gz 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79612.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137329.log 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79612.log.gz 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.47935.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110597.log 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47935.log.gz 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122692.log 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137329.log.gz 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110597.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93699.log 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110597.log.gz 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70175.log 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122692.log: /var/log/ceph/cluster2-client.mirror.93699.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89153.log 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122692.log.gz 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93699.log.gz 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70175.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62134.log 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70175.log.gz 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89153.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.142809.log 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62134.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66544.log 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89153.log.gz 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62134.log.gz 2026-02-20T21:47:15.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.142809.log: gzip 0.0% -5 --verbose -- replaced with /var/log/ceph/cluster1-client.admin.142809.log.gz -- 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr: /var/log/ceph/cluster1-client.mirror.114665.log 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63343.log 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66544.log: /var/log/ceph/cluster1-client.mirror.114665.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113119.log 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66544.log.gz 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63343.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126301.log 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63343.log.gz 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113119.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78717.log 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142235.log 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.126301.log: /var/log/ceph/cluster2-client.mirror.78717.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124617.log 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.126301.log.gz 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.78717.log.gz 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142235.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137127.log 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr: 94.8% -- replaced with /var/log/ceph/cluster1-client.mirror.114665.log.gz 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113119.log.gz 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124617.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.142583.log 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137127.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139835.log 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137127.log.gz 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.142583.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87442.log 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.142583.log.gz 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124617.log.gz 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139835.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113294.log 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.139835.log.gz 2026-02-20T21:47:15.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115207.log 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113294.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98266.log 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113294.log.gz 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.115207.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81972.log 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98266.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61634.log 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98266.log.gz 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81972.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85952.log 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61634.log: gzip 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81972.log.gz -5 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.82999.log 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.115207.log.gz 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61634.log.gz 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr:gzip/var/log/ceph/cluster2-client.mirror.85952.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82158.log 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82999.log: 89.1%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.85952.log.gz 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120253.log 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82999.log.gz 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82158.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135202.log 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82158.log.gz 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87442.log.gz 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.120253.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137014.log 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135202.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63029.log 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.135202.log.gz 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137014.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137014.log.gz 2026-02-20T21:47:15.246 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81321.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63029.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96519.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.120253.log.gz 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81321.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.63029.log.gz 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61541.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81321.log.gz 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96519.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72771.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61541.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96519.log.gz 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114401.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61541.log.gz 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72771.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88813.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72771.log.gz 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114401.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136035.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114401.log.gz 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88813.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114587.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136035.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103014.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114587.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103059.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114587.log.gz 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.103014.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60248.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.103014.log.gz 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.103059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85381.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.60248.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.136035.log.gz 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74368.log 2026-02-20T21:47:15.247 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.60248.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85381.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134259.log 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.85381.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.74368.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.103059.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79647.log 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74368.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134259.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56854.log 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr: 93.5%/var/log/ceph/cluster2-client.mirror.79647.log: -- replaced with /var/log/ceph/cluster2-client.mirror.88813.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110379.log 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.134259.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.79647.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56854.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59473.log 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56854.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110379.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80170.log 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.110379.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59473.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48243.log 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59473.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80170.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55884.log 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.48243.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80170.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122486.log 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.48243.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55884.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80449.log 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.55884.log.gz 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64645.log 2026-02-20T21:47:15.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.122486.log: /var/log/ceph/cluster2-client.mirror.80449.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85886.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64645.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80449.log.gz 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94360.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64645.log.gz 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85886.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136210.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.85886.log.gz 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.122486.log.gz 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94360.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80298.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136210.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89069.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80298.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131687.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.136210.log.gz 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80298.log.gz 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89069.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72405.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131687.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78217.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.89069.log.gz 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72405.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128693.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78217.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127335.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.78217.log.gz 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128693.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56466.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131687.log.gz 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.128693.log.gz 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127335.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76504.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127335.log.gz 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56466.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76461.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56466.log.gz 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76504.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79275.log 2026-02-20T21:47:15.249 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104589.log 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76461.log: /var/log/ceph/cluster2-client.mirror.79275.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72045.log 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79275.log.gz 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104589.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.143557.log 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104589.log.gz 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72045.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62041.log 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.143557.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72045.log.gz 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.143557.log.gz 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125826.log 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54236.log/var/log/ceph/cluster2-client.mirror.62041.log: 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125826.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.62041.log.gz 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118688.log 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125826.log.gz 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64052.log 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54236.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76504.log.gz 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118688.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72488.log 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118688.log.gz 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64052.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95617.log 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64052.log.gz 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72488.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84332.log 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76461.log.gz 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72488.log.gz 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95617.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48311.log 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95617.log.gz 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84332.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65858.log 2026-02-20T21:47:15.250 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.84332.log.gz 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48311.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60739.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48311.log.gz 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65858.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105513.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65858.log.gz 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60739.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61913.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60739.log.gz 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105513.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138454.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61913.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89480.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.61913.log.gz 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138454.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60042.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138454.log.gz 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133320.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60042.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120665.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.89480.log.gz 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133320.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105153.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120665.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65765.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120665.log.gz 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105153.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140706.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.105153.log.gz 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73757.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65765.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.133320.log.gz 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140706.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65765.log.gz 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.140706.log.gz 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130044.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73757.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63622.log 2026-02-20T21:47:15.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130044.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120275.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63622.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60042.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112833.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63622.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120275.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105277.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112833.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.130044.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107871.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.112833.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105277.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140104.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105277.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107871.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95759.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107871.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.140104.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97637.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.140104.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95759.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133711.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97637.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86659.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133711.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128998.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86659.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.95759.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128728.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86659.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.97637.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.135872.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.128998.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128728.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88899.log 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.133711.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.128728.log.gz 2026-02-20T21:47:15.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.135872.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77380.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88899.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137262.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77380.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120707.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77380.log.gz 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137262.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99270.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137262.log.gz 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120707.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94668.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.120707.log.gz 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99270.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135660.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99270.log.gz 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94668.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102893.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135660.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61204.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.102893.log: -- /var/log/ceph/cluster2-client.mirror.114216.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88899.log.gz 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61204.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102893.log.gz 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129884.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61204.log.gz 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114216.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.135660.log.gz 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138657.log 88.2% 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.114216.log.gz 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129884.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87350.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138657.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.129884.log.gz 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.135872.log.gzgzip 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125280.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.138657.log.gz 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87350.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114478.log 2026-02-20T21:47:15.253 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87350.log.gz 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125280.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91515.log 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%/var/log/ceph/cluster1-client.mirror.114478.log: -- replaced with /var/log/ceph/cluster2-client.mirror.125280.log.gz 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.142633.log 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91515.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142708.log 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91515.log.gz 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.142633.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.142633.log.gz 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.117895.log 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.114478.log.gz 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142708.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122168.log 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.142708.log.gz 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.117895.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101205.log 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122168.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94078.log 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122168.log.gz 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101205.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99762.log 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101205.log.gz 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94078.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.117895.log.gz 2026-02-20T21:47:15.254 INFO:teuthology.orchestra.run.trial163.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.94078.log.gz 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118522.log 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.99762.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123933.log 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.99762.log.gz 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126234.log 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118522.log: gzip/var/log/ceph/cluster1-client.mirror.123933.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83772.log 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126234.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.123933.log.gz 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88313.log 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126234.log.gz 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.118522.log.gz 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83772.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88426.log 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.83772.log.gz 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88313.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105010.log 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88426.log: gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105010.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135993.log 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.105010.log.gz 2026-02-20T21:47:15.266 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134421.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2.log: /var/log/ceph/cluster2-client.mirror.135993.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71756.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134421.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.135993.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46922.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138004.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71756.log: /var/log/ceph/cluster2-client.admin.46922.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91351.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71756.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46922.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138004.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88313.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123958.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138004.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.134421.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91351.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.91351.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125656.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99390.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123958.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88426.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125656.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109541.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.125656.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104735.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99390.log: /var/log/ceph/cluster2-client.mirror.109541.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.123958.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: 87.2% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.99390.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135476.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.109541.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57983.log/var/log/ceph/cluster1-client.mirror.104735.log: 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135476.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99874.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.104735.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57983.log: gzip 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.135476.log.gz -5 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.79984.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57983.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.99874.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74135.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79984.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70361.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79984.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74135.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128075.log 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74135.log.gz 2026-02-20T21:47:15.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70361.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140189.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70361.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128075.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95532.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128075.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140189.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121467.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95532.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72299.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121467.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.95532.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52535.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121467.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 95.2% -- replaced with /var/log/ceph/cluster1-client.mirror.99874.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72299.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140127.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.140189.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52535.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58309.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.140127.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112925.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.140127.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52535.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58309.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134805.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58309.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112925.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102215.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142753.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134805.log: /var/log/ceph/cluster2-client.mirror.102215.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92283.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.112925.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102215.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.142753.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.134805.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142753.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107375.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92283.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76715.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107375.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113541.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.107375.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.admin.76715.log: -- /var/log/ceph/cluster2-client.mirror.68590.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113541.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61076.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.admin.76715.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68590.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113541.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137082.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68590.log.gz 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61076.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134237.log 2026-02-20T21:47:15.268 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89869.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137082.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61076.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137082.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134237.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98894.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72299.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89869.log: gzip -5 --verbose 89.3% -- -- replaced with /var/log/ceph/cluster2-client.mirror.134237.log.gz /var/log/ceph/cluster2-client.mirror.118066.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108551.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98894.log: /var/log/ceph/cluster2-client.mirror.118066.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139056.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98894.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118066.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108551.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104104.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108551.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139056.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129033.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.139056.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104104.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47383.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104104.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129033.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76822.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.129033.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.47383.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136323.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 83.9% -- replaced with /var/log/ceph/cluster1-client.mirror.47383.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76822.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105471.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136323.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76822.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80635.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105471.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88293.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80635.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56625.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80635.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.136323.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88293.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76112.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56625.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137914.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.105471.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56625.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.76112.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60611.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 77.3% -- replaced with /var/log/ceph/cluster1-client.mirror.76112.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130732.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137914.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137914.log.gz 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60611.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62471.log 2026-02-20T21:47:15.269 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60611.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130732.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109058.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62471.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130732.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133598.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62471.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109058.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137892.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.133598.log: --verbose -- /var/log/ceph/cluster2-client.mirror.130247.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137892.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110833.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137892.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130247.log: 92.6% 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109058.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.133598.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51956.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130247.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110833.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52869.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.110833.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51956.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88293.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69578.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.52869.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89438.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69578.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125396.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51956.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69578.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89438.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.52869.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118263.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85447.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125396.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.89438.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118263.log: gzip -5 87.9% --verbose -- replaced with /var/log/ceph/cluster2-client.mirror.125396.log.gz -- 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: /var/log/ceph/cluster2-client.mirror.72678.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.118263.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85447.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64924.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.85447.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72678.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71891.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72678.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42804.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64924.log: /var/log/ceph/cluster2-client.mirror.71891.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72736.log 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.64924.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.42804.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71891.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.42804.log.gz 2026-02-20T21:47:15.270 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77694.log 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64610.log 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72736.log: /var/log/ceph/cluster2-client.mirror.77694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140685.log 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72736.log.gz 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64610.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77694.log.gz 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98929.log 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64610.log.gz 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140685.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55302.log 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.140685.log.gz 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.98929.log: /var/log/ceph/cluster1-client.mirror.3.142378.log 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55302.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.45893.log 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98929.log.gz 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55302.log.gz 2026-02-20T21:47:15.271 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.3.142378.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141949.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.45893.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118723.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.45893.log.gz 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141949.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83278.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118723.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.142378.log.gz 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73810.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118723.log.gz 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52990.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83278.log: /var/log/ceph/cluster1-client.admin.73810.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87978.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.83278.log.gz 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.73810.log.gz 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.52990.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135322.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87978.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.138828.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135322.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.87978.log.gz 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.52990.log.gz 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78496.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.135322.log.gz 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:gzip/var/log/ceph/cluster2-client.mirror.138828.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132960.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78496.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.45779.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78496.log.gz 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132960.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55011.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.45779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132687.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.45779.log.gz 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55011.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128342.log 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55011.log.gz 2026-02-20T21:47:15.274 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122560.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132687.log: /var/log/ceph/cluster2-client.mirror.128342.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.132960.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119448.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122560.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126326.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122560.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.119448.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95447.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130282.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.132687.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95447.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82871.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.95447.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130282.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95779.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130282.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82871.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66137.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.126326.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.119448.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82871.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.95779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100126.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66137.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58891.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66137.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141685.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58891.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100126.log.gzgzip 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83464.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58891.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.141685.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.95779.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140074.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr: 85.0% -- replaced with /var/log/ceph/cluster1-client.mirror.141685.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83464.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60539.log 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.83464.log.gz 2026-02-20T21:47:15.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140074.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137059.log 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129549.log 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.60539.log: /var/log/ceph/cluster1-client.mirror.137059.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.84203.log 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.60539.log.gz 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137059.log.gz 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.140074.log.gz 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.117850.log 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.129549.log.gz 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.84203.log: -- /var/log/ceph/cluster2-client.mirror.72157.log 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.117850.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104780.log 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr: 93.6%/var/log/ceph/cluster2-client.mirror.72157.log: -- replaced with /var/log/ceph/cluster1-client.mirror.84203.log.gz 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121236.log 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.117850.log.gz 2026-02-20T21:47:15.276 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72157.log.gz 2026-02-20T21:47:15.277 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.104780.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115762.log 2026-02-20T21:47:15.277 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99829.log 2026-02-20T21:47:15.277 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121236.log: gzip/var/log/ceph/cluster2-client.mirror.115762.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108640.log 2026-02-20T21:47:15.277 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121236.log.gz 2026-02-20T21:47:15.277 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99829.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70919.log 2026-02-20T21:47:15.277 INFO:teuthology.orchestra.run.trial163.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.104780.log.gz 2026-02-20T21:47:15.277 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67718.log 2026-02-20T21:47:15.277 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.108640.log.gz 2026-02-20T21:47:15.277 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70919.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70454.log 2026-02-20T21:47:15.277 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70919.log.gz/var/log/ceph/cluster2-client.mirror.67718.log: 2026-02-20T21:47:15.278 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130821.log 2026-02-20T21:47:15.278 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70454.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67718.log.gz 2026-02-20T21:47:15.278 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101474.log 2026-02-20T21:47:15.278 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70454.log.gz 2026-02-20T21:47:15.278 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130821.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135780.log 2026-02-20T21:47:15.278 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130821.log.gz 2026-02-20T21:47:15.278 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119357.log 2026-02-20T21:47:15.279 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101474.log: /var/log/ceph/cluster2-client.mirror.135780.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99994.log 2026-02-20T21:47:15.279 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119357.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138274.log 2026-02-20T21:47:15.279 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119357.log.gz 2026-02-20T21:47:15.279 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99994.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92203.log 2026-02-20T21:47:15.279 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.99994.log.gz 2026-02-20T21:47:15.280 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138274.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101582.log 2026-02-20T21:47:15.280 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.138274.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.135780.log.gz 2026-02-20T21:47:15.280 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:15.280 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92203.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114033.log 2026-02-20T21:47:15.280 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101582.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54976.log 2026-02-20T21:47:15.280 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.101582.log.gz 2026-02-20T21:47:15.281 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114033.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95667.log 2026-02-20T21:47:15.281 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114033.log.gz 2026-02-20T21:47:15.281 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54976.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46971.log 2026-02-20T21:47:15.281 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54976.log.gz 2026-02-20T21:47:15.281 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95667.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105616.log 2026-02-20T21:47:15.282 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%/var/log/ceph/cluster2-client.admin.46971.log: -- replaced with /var/log/ceph/cluster2-client.mirror.95667.log.gz 2026-02-20T21:47:15.282 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135547.log 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46971.log.gz 2026-02-20T21:47:15.282 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:15.282 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69896.log 2026-02-20T21:47:15.282 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105616.log.gz 2026-02-20T21:47:15.282 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135547.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.142085.log 2026-02-20T21:47:15.282 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69896.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116434.log 2026-02-20T21:47:15.283 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.135547.log.gz 2026-02-20T21:47:15.283 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69896.log.gz 2026-02-20T21:47:15.283 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.142085.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66823.log 2026-02-20T21:47:15.283 INFO:teuthology.orchestra.run.trial163.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.142085.log.gz 2026-02-20T21:47:15.283 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.116434.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118874.log 2026-02-20T21:47:15.283 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.116434.log.gz 2026-02-20T21:47:15.284 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66823.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.143165.log 2026-02-20T21:47:15.284 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66823.log.gz 2026-02-20T21:47:15.284 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118874.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91589.log 2026-02-20T21:47:15.284 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.118874.log.gz/var/log/ceph/cluster2-client.admin.143165.log: 2026-02-20T21:47:15.285 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.143165.log.gz 2026-02-20T21:47:15.285 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46388.log 2026-02-20T21:47:15.285 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91589.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68032.log 2026-02-20T21:47:15.285 INFO:teuthology.orchestra.run.trial163.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91589.log.gz 2026-02-20T21:47:15.285 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.46388.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46388.log.gz 2026-02-20T21:47:15.285 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74016.log 2026-02-20T21:47:15.286 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68032.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111375.log 2026-02-20T21:47:15.286 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68032.log.gz 2026-02-20T21:47:15.286 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74016.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95251.log 2026-02-20T21:47:15.286 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74016.log.gz 2026-02-20T21:47:15.286 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111375.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136432.log 2026-02-20T21:47:15.287 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95251.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135180.log 2026-02-20T21:47:15.287 INFO:teuthology.orchestra.run.trial163.stderr:gzip/var/log/ceph/cluster1-client.mirror.136432.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100557.log 2026-02-20T21:47:15.287 INFO:teuthology.orchestra.run.trial163.stderr: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.136432.log.gz 2026-02-20T21:47:15.287 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135180.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140747.log 2026-02-20T21:47:15.287 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100557.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.135180.log.gz 2026-02-20T21:47:15.288 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100557.log.gz 2026-02-20T21:47:15.288 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.136398.log 2026-02-20T21:47:15.288 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140747.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137622.log 2026-02-20T21:47:15.288 INFO:teuthology.orchestra.run.trial163.stderr: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.140747.log.gz 2026-02-20T21:47:15.289 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136398.log: gzip -5 --verbose 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.111375.log.gz 2026-02-20T21:47:15.289 INFO:teuthology.orchestra.run.trial163.stderr: -- /var/log/ceph/cluster2-client.mirror.79798.log 2026-02-20T21:47:15.289 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137622.log: gzip -5 --verbose 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.137622.log.gz 2026-02-20T21:47:15.289 INFO:teuthology.orchestra.run.trial163.stderr: -- /var/log/ceph/cluster2-client.mirror.65510.log 2026-02-20T21:47:15.290 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79798.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95346.log 2026-02-20T21:47:15.290 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79798.log.gz 2026-02-20T21:47:15.290 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65510.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.136398.log.gz 2026-02-20T21:47:15.290 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46873.log 2026-02-20T21:47:15.290 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95346.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101340.log 2026-02-20T21:47:15.290 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95346.log.gz 2026-02-20T21:47:15.291 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.46873.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46873.log.gz 2026-02-20T21:47:15.291 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93021.log 2026-02-20T21:47:15.291 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101340.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117359.log 2026-02-20T21:47:15.291 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101340.log.gz 2026-02-20T21:47:15.291 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93021.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86997.log 2026-02-20T21:47:15.292 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103545.log 2026-02-20T21:47:15.292 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117359.log.gz 2026-02-20T21:47:15.292 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86997.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78682.log 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86997.log.gz 2026-02-20T21:47:15.292 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:15.293 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103545.log: gzip -5 --verbose -- 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103545.log.gz 2026-02-20T21:47:15.293 INFO:teuthology.orchestra.run.trial163.stderr: /var/log/ceph/cluster2-client.mirror.81007.log 2026-02-20T21:47:15.293 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78682.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47748.log 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78682.log.gz 2026-02-20T21:47:15.293 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:15.294 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81007.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136519.log 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81007.log.gz 2026-02-20T21:47:15.294 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:15.294 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.47748.log: gzip -5 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.47748.log.gz 2026-02-20T21:47:15.294 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.90291.log 2026-02-20T21:47:15.294 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136519.log: gzip 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.136519.log.gz 2026-02-20T21:47:15.294 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110696.log 2026-02-20T21:47:15.295 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.90291.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137644.log 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.90291.log.gz 2026-02-20T21:47:15.295 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:15.295 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110696.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107456.log 2026-02-20T21:47:15.296 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137644.log: 86.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89624.log 2026-02-20T21:47:15.296 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.137644.log.gz 2026-02-20T21:47:15.296 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107456.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60152.log 2026-02-20T21:47:15.296 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89624.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107456.log.gz 2026-02-20T21:47:15.297 INFO:teuthology.orchestra.run.trial163.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.89624.log.gz 2026-02-20T21:47:15.297 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138319.log 2026-02-20T21:47:15.297 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122245.log 2026-02-20T21:47:15.297 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.60152.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60152.log.gz 2026-02-20T21:47:15.298 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138319.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138319.log.gz 2026-02-20T21:47:15.298 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78810.log 2026-02-20T21:47:15.298 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77787.log 2026-02-20T21:47:15.299 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122245.log: /var/log/ceph/cluster2-client.mirror.78810.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.122245.log.gz 2026-02-20T21:47:15.299 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78810.log.gz 2026-02-20T21:47:15.299 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65672.log 2026-02-20T21:47:15.299 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99010.log 2026-02-20T21:47:15.300 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77787.log: /var/log/ceph/cluster2-client.mirror.65672.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77787.log.gz 2026-02-20T21:47:15.300 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65672.log.gz 2026-02-20T21:47:15.300 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91009.log 2026-02-20T21:47:15.300 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122595.log 2026-02-20T21:47:15.301 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99010.log: /var/log/ceph/cluster2-client.mirror.91009.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.99010.log.gz 2026-02-20T21:47:15.301 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50225.log 2026-02-20T21:47:15.302 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122595.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122595.log.gz 2026-02-20T21:47:15.302 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87723.log 2026-02-20T21:47:15.303 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50225.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86053.log 2026-02-20T21:47:15.303 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50225.log.gz 2026-02-20T21:47:15.303 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87723.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87723.log.gz 2026-02-20T21:47:15.304 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83660.log 2026-02-20T21:47:15.304 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86053.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.86053.log.gz 2026-02-20T21:47:15.305 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84736.log 2026-02-20T21:47:15.305 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83660.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.83660.log.gz 2026-02-20T21:47:15.305 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115822.log 2026-02-20T21:47:15.306 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84736.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84736.log.gz 2026-02-20T21:47:15.306 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136879.log 2026-02-20T21:47:15.307 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115822.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115822.log.gz 2026-02-20T21:47:15.307 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51007.log 2026-02-20T21:47:15.308 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136879.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136879.log.gz 2026-02-20T21:47:15.308 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70977.log 2026-02-20T21:47:15.308 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51007.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135397.log 2026-02-20T21:47:15.309 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51007.log.gz 2026-02-20T21:47:15.309 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70977.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70977.log.gz 2026-02-20T21:47:15.310 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138544.log 2026-02-20T21:47:15.310 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135397.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.135397.log.gz 2026-02-20T21:47:15.311 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139856.log 2026-02-20T21:47:15.311 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138544.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.138544.log.gz 2026-02-20T21:47:15.311 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89846.log 2026-02-20T21:47:15.312 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139856.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.139856.log.gz 2026-02-20T21:47:15.312 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90914.log 2026-02-20T21:47:15.313 INFO:teuthology.orchestra.run.trial069.stderr: 91.0% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-20T21:47:15.313 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89846.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.89846.log.gz 2026-02-20T21:47:15.313 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112150.log 2026-02-20T21:47:15.314 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90914.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.90914.log.gz 2026-02-20T21:47:15.314 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141370.log 2026-02-20T21:47:15.315 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112150.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.112150.log.gz 2026-02-20T21:47:15.316 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125361.log 2026-02-20T21:47:15.316 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.141370.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.141370.log.gz 2026-02-20T21:47:15.316 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128029.log 2026-02-20T21:47:15.317 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125361.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125361.log.gz 2026-02-20T21:47:15.317 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55946.log 2026-02-20T21:47:15.318 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128029.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128029.log.gz 2026-02-20T21:47:15.318 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107836.log 2026-02-20T21:47:15.319 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55946.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55946.log.gz 2026-02-20T21:47:15.319 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71628.log 2026-02-20T21:47:15.319 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107836.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107836.log.gz 2026-02-20T21:47:15.320 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76774.log 2026-02-20T21:47:15.320 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71628.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71628.log.gz 2026-02-20T21:47:15.321 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103763.log 2026-02-20T21:47:15.331 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76774.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55170.log 2026-02-20T21:47:15.331 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103763.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103763.log.gz 2026-02-20T21:47:15.338 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137442.log 2026-02-20T21:47:15.341 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55170.log: 88.4% 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.115762.log.gz 2026-02-20T21:47:15.341 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55399.log 2026-02-20T21:47:15.342 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137442.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137442.log.gz 2026-02-20T21:47:15.342 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.55170.log.gz 2026-02-20T21:47:15.342 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102418.log 2026-02-20T21:47:15.343 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55399.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55399.log.gz 2026-02-20T21:47:15.343 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54138.log 2026-02-20T21:47:15.359 INFO:teuthology.orchestra.run.trial069.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-20T21:47:15.367 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102418.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64273.log 2026-02-20T21:47:15.375 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54138.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88512.log 2026-02-20T21:47:15.384 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64273.log: 88.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121143.log 2026-02-20T21:47:15.384 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.64273.log.gz 2026-02-20T21:47:15.394 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88512.log: 93.6%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112945.log 2026-02-20T21:47:15.400 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121143.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.88512.log.gz 2026-02-20T21:47:15.402 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130339.log 2026-02-20T21:47:15.406 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.112945.log: -- replaced with /var/log/ceph/cluster2-client.mirror.121143.log.gz 2026-02-20T21:47:15.411 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.140550.log 2026-02-20T21:47:15.416 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130339.log: 92.4% 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.112945.log.gz 2026-02-20T21:47:15.422 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80484.log 2026-02-20T21:47:15.425 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.1.140550.log: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.130339.log.gz 2026-02-20T21:47:15.433 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138409.log 2026-02-20T21:47:15.434 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80484.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.80484.log.gz 2026-02-20T21:47:15.435 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.105513.log.gz 2026-02-20T21:47:15.438 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.1.140550.log.gz 2026-02-20T21:47:15.444 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86391.log 2026-02-20T21:47:15.445 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138409.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138409.log.gz 2026-02-20T21:47:15.454 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76615.log 2026-02-20T21:47:15.463 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86391.log: 89.1%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69113.log 2026-02-20T21:47:15.466 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76615.log: -- replaced with /var/log/ceph/cluster2-client.mirror.86391.log.gz 2026-02-20T21:47:15.473 INFO:teuthology.orchestra.run.trial163.stderr: 92.1%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90130.log 2026-02-20T21:47:15.473 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69113.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69113.log.gz 2026-02-20T21:47:15.483 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.76615.log.gz 2026-02-20T21:47:15.485 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75607.log 2026-02-20T21:47:15.485 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90130.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90130.log.gz 2026-02-20T21:47:15.495 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96713.log 2026-02-20T21:47:15.496 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75607.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75607.log.gz 2026-02-20T21:47:15.505 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62192.log 2026-02-20T21:47:15.511 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96713.log: 87.9% 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.110696.log.gz 2026-02-20T21:47:15.512 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.96713.log.gz 2026-02-20T21:47:15.512 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64552.log 2026-02-20T21:47:15.512 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62192.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62192.log.gz 2026-02-20T21:47:15.512 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54030.log 2026-02-20T21:47:15.512 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64552.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64552.log.gz 2026-02-20T21:47:15.512 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117969.log 2026-02-20T21:47:15.512 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54030.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62692.log 2026-02-20T21:47:15.515 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.54030.log.gz 2026-02-20T21:47:15.516 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117969.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117969.log.gz 2026-02-20T21:47:15.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132477.log 2026-02-20T21:47:15.517 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62692.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62692.log.gz 2026-02-20T21:47:15.517 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124363.log 2026-02-20T21:47:15.520 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132477.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.132477.log.gz 2026-02-20T21:47:15.521 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74642.log 2026-02-20T21:47:15.521 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124363.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124363.log.gz 2026-02-20T21:47:15.521 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134330.log 2026-02-20T21:47:15.521 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74642.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74642.log.gz 2026-02-20T21:47:15.521 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104209.log 2026-02-20T21:47:15.521 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134330.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134330.log.gz 2026-02-20T21:47:15.533 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113685.log 2026-02-20T21:47:15.533 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104209.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.104209.log.gz 2026-02-20T21:47:15.533 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63808.log 2026-02-20T21:47:15.533 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.113685.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.113685.log.gz 2026-02-20T21:47:15.533 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46241.log 2026-02-20T21:47:15.533 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63808.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63808.log.gz 2026-02-20T21:47:15.533 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61355.log 2026-02-20T21:47:15.533 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.46241.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46241.log.gz 2026-02-20T21:47:15.533 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124911.log 2026-02-20T21:47:15.534 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61355.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61355.log.gz 2026-02-20T21:47:15.534 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136564.log 2026-02-20T21:47:15.534 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124911.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.124911.log.gz 2026-02-20T21:47:15.534 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141581.log 2026-02-20T21:47:15.534 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136564.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.136564.log.gz 2026-02-20T21:47:15.534 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61111.log 2026-02-20T21:47:15.534 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.141581.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.141581.log.gz 2026-02-20T21:47:15.534 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51326.log 2026-02-20T21:47:15.534 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61111.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.61111.log.gz 2026-02-20T21:47:15.534 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84841.log 2026-02-20T21:47:15.534 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81135.log 2026-02-20T21:47:15.534 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51326.log.gz 2026-02-20T21:47:15.535 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84841.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84841.log.gz 2026-02-20T21:47:15.535 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112601.log 2026-02-20T21:47:15.535 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81135.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81135.log.gz 2026-02-20T21:47:15.535 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101683.log 2026-02-20T21:47:15.538 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112601.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112601.log.gz 2026-02-20T21:47:15.538 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73050.log 2026-02-20T21:47:15.538 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101683.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101683.log.gz 2026-02-20T21:47:15.538 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93295.log 2026-02-20T21:47:15.538 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73050.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73050.log.gz 2026-02-20T21:47:15.538 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137487.log 2026-02-20T21:47:15.541 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93295.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54236.log.gz 2026-02-20T21:47:15.543 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46628.log 2026-02-20T21:47:15.543 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137487.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137487.log.gz 2026-02-20T21:47:15.543 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76857.log 2026-02-20T21:47:15.544 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.46628.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46628.log.gz 2026-02-20T21:47:15.544 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86694.log 2026-02-20T21:47:15.545 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76857.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.76857.log.gz 2026-02-20T21:47:15.546 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99067.log 2026-02-20T21:47:15.546 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86694.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86694.log.gz 2026-02-20T21:47:15.546 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135109.log 2026-02-20T21:47:15.547 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99067.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.99067.log.gz 2026-02-20T21:47:15.547 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104678.log 2026-02-20T21:47:15.548 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135109.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.135109.log.gz 2026-02-20T21:47:15.548 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133094.log 2026-02-20T21:47:15.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104678.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.104678.log.gz 2026-02-20T21:47:15.550 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.120275.log.gz 2026-02-20T21:47:15.550 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94965.log 2026-02-20T21:47:15.550 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106949.log 2026-02-20T21:47:15.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133094.log: /var/log/ceph/cluster2-client.mirror.94965.log: 88.4% 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94965.log.gz 2026-02-20T21:47:15.550 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.133094.log.gz 2026-02-20T21:47:15.554 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67009.log 2026-02-20T21:47:15.555 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.106949.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88184.log 2026-02-20T21:47:15.555 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.106949.log.gz 2026-02-20T21:47:15.555 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67009.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140333.log 2026-02-20T21:47:15.555 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88184.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67009.log.gz 2026-02-20T21:47:15.555 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128110.log 2026-02-20T21:47:15.555 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.140333.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.140333.log.gz 2026-02-20T21:47:15.555 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134918.log 2026-02-20T21:47:15.555 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128110.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128110.log.gz 2026-02-20T21:47:15.555 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99355.log 2026-02-20T21:47:15.560 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134918.log: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.88184.log.gz 2026-02-20T21:47:15.560 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113948.log 2026-02-20T21:47:15.560 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99355.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.134918.log.gz 2026-02-20T21:47:15.560 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99355.log.gz 2026-02-20T21:47:15.560 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73779.log 2026-02-20T21:47:15.560 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68962.log 2026-02-20T21:47:15.561 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113948.log: /var/log/ceph/cluster1-client.admin.73779.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113948.log.gz 2026-02-20T21:47:15.561 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.admin.73779.log.gz 2026-02-20T21:47:15.561 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101901.log 2026-02-20T21:47:15.561 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82065.log 2026-02-20T21:47:15.561 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68962.log: /var/log/ceph/cluster2-client.mirror.101901.log: 88.7% 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.101901.log.gz 2026-02-20T21:47:15.561 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.68962.log.gz 2026-02-20T21:47:15.561 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.52914.log 2026-02-20T21:47:15.561 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82065.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106123.log 2026-02-20T21:47:15.561 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82065.log.gz 2026-02-20T21:47:15.561 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.52914.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.91426.log 2026-02-20T21:47:15.561 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106123.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106123.log.gz 2026-02-20T21:47:15.562 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113329.log 2026-02-20T21:47:15.564 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.91426.log: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.52914.log.gz 2026-02-20T21:47:15.564 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136789.log 2026-02-20T21:47:15.565 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113329.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113329.log.gz 2026-02-20T21:47:15.565 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72322.log 2026-02-20T21:47:15.566 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136789.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.136789.log.gz 2026-02-20T21:47:15.566 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129952.log 2026-02-20T21:47:15.567 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72322.log: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.0.91426.log.gz 2026-02-20T21:47:15.568 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.45942.log 2026-02-20T21:47:15.568 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129952.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.129952.log.gz 2026-02-20T21:47:15.570 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82778.log 2026-02-20T21:47:15.570 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.45942.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.45942.log.gz 2026-02-20T21:47:15.570 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87622.log 2026-02-20T21:47:15.570 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82778.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82778.log.gz 2026-02-20T21:47:15.571 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139121.log 2026-02-20T21:47:15.571 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87622.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87622.log.gz 2026-02-20T21:47:15.572 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97115.log 2026-02-20T21:47:15.572 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139121.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.139121.log.gz 2026-02-20T21:47:15.573 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114500.log 2026-02-20T21:47:15.573 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97115.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97115.log.gz 2026-02-20T21:47:15.573 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115086.log 2026-02-20T21:47:15.593 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114500.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.143361.log 2026-02-20T21:47:15.593 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115086.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115086.log.gz 2026-02-20T21:47:15.605 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82123.log 2026-02-20T21:47:15.605 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.143361.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.143361.log.gz 2026-02-20T21:47:15.612 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127168.log 2026-02-20T21:47:15.613 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82123.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82123.log.gz 2026-02-20T21:47:15.621 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136992.log 2026-02-20T21:47:15.622 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127168.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127168.log.gz 2026-02-20T21:47:15.627 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73757.log.gz 2026-02-20T21:47:15.627 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92933.log 2026-02-20T21:47:15.628 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136992.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.136992.log.gz 2026-02-20T21:47:15.628 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133060.log 2026-02-20T21:47:15.637 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92933.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77043.log 2026-02-20T21:47:15.650 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133060.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133060.log.gz 2026-02-20T21:47:15.652 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139938.log 2026-02-20T21:47:15.653 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77043.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77043.log.gz 2026-02-20T21:47:15.659 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50342.log 2026-02-20T21:47:15.661 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.128342.log.gz 2026-02-20T21:47:15.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139938.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.139938.log.gz 2026-02-20T21:47:15.662 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53973.log 2026-02-20T21:47:15.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50342.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50342.log.gz 2026-02-20T21:47:15.662 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110421.log 2026-02-20T21:47:15.667 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53973.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53973.log.gz 2026-02-20T21:47:15.667 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137374.log 2026-02-20T21:47:15.668 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110421.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110421.log.gz 2026-02-20T21:47:15.668 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129849.log 2026-02-20T21:47:15.668 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137374.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137374.log.gz 2026-02-20T21:47:15.668 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134563.log 2026-02-20T21:47:15.668 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129849.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.129849.log.gz 2026-02-20T21:47:15.668 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137869.log 2026-02-20T21:47:15.668 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134563.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.134563.log.gz 2026-02-20T21:47:15.668 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139537.log 2026-02-20T21:47:15.668 INFO:teuthology.orchestra.run.trial163.stderr: 95.5%/var/log/ceph/cluster1-client.mirror.137869.log: -- replaced with /var/log/ceph/cluster2-client.mirror.99829.log.gz 2026-02-20T21:47:15.668 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137869.log.gz 2026-02-20T21:47:15.669 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46481.log 2026-02-20T21:47:15.669 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5/var/log/ceph/cluster2-client.mirror.139537.log: --verbose -- /var/log/ceph/cluster2-client.mirror.65059.log 2026-02-20T21:47:15.679 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.46481.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.139537.log.gz 2026-02-20T21:47:15.679 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46481.log.gz 2026-02-20T21:47:15.679 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72864.log 2026-02-20T21:47:15.679 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65059.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115857.log 2026-02-20T21:47:15.679 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65059.log.gz 2026-02-20T21:47:15.680 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72864.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72864.log.gz 2026-02-20T21:47:15.680 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130449.log 2026-02-20T21:47:15.680 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115857.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88063.log 2026-02-20T21:47:15.680 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.115857.log.gz 2026-02-20T21:47:15.680 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130449.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59182.log 2026-02-20T21:47:15.680 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130449.log.gz 2026-02-20T21:47:15.680 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88063.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88063.log.gz 2026-02-20T21:47:15.680 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119555.log 2026-02-20T21:47:15.680 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59182.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99560.log 2026-02-20T21:47:15.680 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59182.log.gz 2026-02-20T21:47:15.680 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69148.log 2026-02-20T21:47:15.681 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99560.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119555.log.gz 2026-02-20T21:47:15.681 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99560.log.gz 2026-02-20T21:47:15.681 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106400.log 2026-02-20T21:47:15.681 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69148.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69148.log.gz 2026-02-20T21:47:15.681 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124236.log 2026-02-20T21:47:15.681 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106400.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.106400.log.gz 2026-02-20T21:47:15.681 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123824.log 2026-02-20T21:47:15.681 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124236.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130484.log 2026-02-20T21:47:15.681 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124236.log.gz 2026-02-20T21:47:15.681 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123824.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104643.log 2026-02-20T21:47:15.681 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130484.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.123824.log.gz 2026-02-20T21:47:15.681 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.130484.log.gz 2026-02-20T21:47:15.682 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103081.log 2026-02-20T21:47:15.682 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104643.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104643.log.gz 2026-02-20T21:47:15.682 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103830.log 2026-02-20T21:47:15.694 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103081.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85179.log 2026-02-20T21:47:15.694 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.103830.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.103830.log.gz 2026-02-20T21:47:15.694 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66602.log 2026-02-20T21:47:15.694 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85179.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.85179.log.gz 2026-02-20T21:47:15.694 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118789.log 2026-02-20T21:47:15.694 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66602.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66602.log.gz 2026-02-20T21:47:15.695 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132616.log 2026-02-20T21:47:15.695 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118789.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118789.log.gz 2026-02-20T21:47:15.697 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52166.log 2026-02-20T21:47:15.698 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132616.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.132616.log.gz 2026-02-20T21:47:15.698 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78961.log 2026-02-20T21:47:15.698 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52166.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80577.log 2026-02-20T21:47:15.698 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52166.log.gz 2026-02-20T21:47:15.701 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78961.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78961.log.gz 2026-02-20T21:47:15.701 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89048.log 2026-02-20T21:47:15.701 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80577.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80577.log.gz 2026-02-20T21:47:15.701 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108243.log 2026-02-20T21:47:15.701 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89048.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.89048.log.gz 2026-02-20T21:47:15.702 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76215.log 2026-02-20T21:47:15.702 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108243.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108243.log.gz 2026-02-20T21:47:15.705 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106793.log 2026-02-20T21:47:15.705 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76215.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76215.log.gz 2026-02-20T21:47:15.705 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132574.log 2026-02-20T21:47:15.706 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106793.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106793.log.gz 2026-02-20T21:47:15.706 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.126504.log 2026-02-20T21:47:15.706 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132574.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132574.log.gz 2026-02-20T21:47:15.706 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52959.log 2026-02-20T21:47:15.714 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.3.126504.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88684.log 2026-02-20T21:47:15.716 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52959.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.101474.log.gz 2026-02-20T21:47:15.728 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.3.126504.log.gz 2026-02-20T21:47:15.729 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46093.log 2026-02-20T21:47:15.732 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88684.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88684.log.gz 2026-02-20T21:47:15.732 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.84001.log 2026-02-20T21:47:15.733 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.46093.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46093.log.gz 2026-02-20T21:47:15.733 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.90291.log 2026-02-20T21:47:15.734 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.84001.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.84001.log.gz 2026-02-20T21:47:15.734 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140169.log 2026-02-20T21:47:15.739 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.90291.log: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.0.90291.log.gz 2026-02-20T21:47:15.739 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110629.log 2026-02-20T21:47:15.740 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140169.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.140169.log.gz 2026-02-20T21:47:15.740 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113812.log 2026-02-20T21:47:15.740 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.110629.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.110629.log.gz 2026-02-20T21:47:15.742 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84569.log 2026-02-20T21:47:15.742 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113812.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.113812.log.gz 2026-02-20T21:47:15.742 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83371.log 2026-02-20T21:47:15.745 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84569.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.84569.log.gz 2026-02-20T21:47:15.745 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95007.log 2026-02-20T21:47:15.745 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83371.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.83371.log.gz 2026-02-20T21:47:15.745 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55364.log 2026-02-20T21:47:15.745 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.95007.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.95007.log.gz 2026-02-20T21:47:15.745 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-20T21:47:15.745 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55364.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.55364.log.gz 2026-02-20T21:47:15.746 INFO:teuthology.orchestra.run.trial069.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-20T21:47:15.747 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53386.log 2026-02-20T21:47:15.747 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138477.log 2026-02-20T21:47:15.748 INFO:teuthology.orchestra.run.trial163.stderr: 92.9% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-20T21:47:15.749 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53386.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110344.log 2026-02-20T21:47:15.750 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53386.log.gz 2026-02-20T21:47:15.751 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138477.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.138477.log.gz 2026-02-20T21:47:15.751 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79368.log 2026-02-20T21:47:15.752 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110344.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.110344.log.gz 2026-02-20T21:47:15.752 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84977.log 2026-02-20T21:47:15.752 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79368.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79368.log.gz 2026-02-20T21:47:15.753 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88618.log 2026-02-20T21:47:15.753 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84977.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.84977.log.gz 2026-02-20T21:47:15.754 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113499.log 2026-02-20T21:47:15.756 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88618.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91908.log.gz 2026-02-20T21:47:15.760 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81379.log 2026-02-20T21:47:15.760 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113499.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113499.log.gz 2026-02-20T21:47:15.760 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88618.log.gz 2026-02-20T21:47:15.760 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103039.log 2026-02-20T21:47:15.760 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137824.log 2026-02-20T21:47:15.760 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81379.log: /var/log/ceph/cluster2-client.mirror.103039.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81379.log.gz 2026-02-20T21:47:15.761 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.53142.log 2026-02-20T21:47:15.761 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137824.log: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.103039.log.gz 2026-02-20T21:47:15.761 INFO:teuthology.orchestra.run.trial163.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137824.log.gz 2026-02-20T21:47:15.761 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78066.log 2026-02-20T21:47:15.761 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.3.53142.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118909.log 2026-02-20T21:47:15.766 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78066.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78066.log.gz 2026-02-20T21:47:15.766 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119392.log 2026-02-20T21:47:15.766 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118909.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.118909.log.gz 2026-02-20T21:47:15.766 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142378.log 2026-02-20T21:47:15.766 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119392.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119392.log.gz 2026-02-20T21:47:15.766 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136086.log 2026-02-20T21:47:15.766 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142378.log: 84.8% -- replaced with /var/log/ceph/cluster2-client.mirror.142378.log.gz 2026-02-20T21:47:15.766 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67253.log 2026-02-20T21:47:15.766 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136086.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.136086.log.gz 2026-02-20T21:47:15.766 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89111.log 2026-02-20T21:47:15.766 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67253.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67253.log.gz 2026-02-20T21:47:15.766 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96309.log 2026-02-20T21:47:15.767 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89111.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89111.log.gz 2026-02-20T21:47:15.768 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79740.log 2026-02-20T21:47:15.769 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96309.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.96309.log.gz 2026-02-20T21:47:15.769 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75944.log 2026-02-20T21:47:15.770 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79740.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79740.log.gz 2026-02-20T21:47:15.770 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97026.log 2026-02-20T21:47:15.771 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75944.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75944.log.gz 2026-02-20T21:47:15.771 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66358.log 2026-02-20T21:47:15.771 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97026.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.97026.log.gz 2026-02-20T21:47:15.772 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140049.log 2026-02-20T21:47:15.773 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66358.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66358.log.gz 2026-02-20T21:47:15.773 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57013.log 2026-02-20T21:47:15.773 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.140049.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.140049.log.gz 2026-02-20T21:47:15.774 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121844.log 2026-02-20T21:47:15.774 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57013.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57013.log.gz 2026-02-20T21:47:15.774 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.135872.log 2026-02-20T21:47:15.775 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121844.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121844.log.gz 2026-02-20T21:47:15.776 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53938.log 2026-02-20T21:47:15.776 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.135872.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.135872.log.gz 2026-02-20T21:47:15.776 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113998.log 2026-02-20T21:47:15.777 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53938.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56951.log 2026-02-20T21:47:15.777 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53938.log.gz 2026-02-20T21:47:15.778 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113998.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113998.log.gz 2026-02-20T21:47:15.778 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136902.log 2026-02-20T21:47:15.779 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56951.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56951.log.gz 2026-02-20T21:47:15.779 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109144.log 2026-02-20T21:47:15.780 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136902.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.136902.log.gz 2026-02-20T21:47:15.780 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58697.log 2026-02-20T21:47:15.781 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.109144.log: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.109144.log.gz 2026-02-20T21:47:15.781 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142127.log 2026-02-20T21:47:15.782 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58697.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58697.log.gz 2026-02-20T21:47:15.782 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83243.log 2026-02-20T21:47:15.783 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142127.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.142127.log.gz 2026-02-20T21:47:15.783 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53730.log 2026-02-20T21:47:15.784 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83243.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.83243.log.gz 2026-02-20T21:47:15.784 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128156.log 2026-02-20T21:47:15.785 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53730.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53730.log.gz 2026-02-20T21:47:15.785 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134847.log 2026-02-20T21:47:15.786 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128156.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.128156.log.gz 2026-02-20T21:47:15.786 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141633.log 2026-02-20T21:47:15.787 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134847.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.134847.log.gz 2026-02-20T21:47:15.788 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105581.log 2026-02-20T21:47:15.789 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141633.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.141633.log.gz 2026-02-20T21:47:15.789 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87943.log 2026-02-20T21:47:15.790 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105581.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.105581.log.gz 2026-02-20T21:47:15.790 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131458.log 2026-02-20T21:47:15.791 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87943.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87943.log.gz 2026-02-20T21:47:15.791 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122049.log 2026-02-20T21:47:15.792 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131458.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131458.log.gz 2026-02-20T21:47:15.793 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120929.log 2026-02-20T21:47:15.793 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122049.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122049.log.gz 2026-02-20T21:47:15.793 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141097.log 2026-02-20T21:47:15.794 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.120929.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.120929.log.gz 2026-02-20T21:47:15.795 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87688.log 2026-02-20T21:47:15.795 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141097.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.141097.log.gz 2026-02-20T21:47:15.796 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136252.log 2026-02-20T21:47:15.796 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87688.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.87688.log.gz 2026-02-20T21:47:15.797 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91984.log 2026-02-20T21:47:15.798 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136252.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.136252.log.gz 2026-02-20T21:47:15.799 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130767.log 2026-02-20T21:47:15.807 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91984.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117917.log 2026-02-20T21:47:15.812 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130767.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.114500.log.gz 2026-02-20T21:47:15.812 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82592.log 2026-02-20T21:47:15.816 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117917.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72405.log.gz 2026-02-20T21:47:15.816 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57304.log 2026-02-20T21:47:15.817 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82592.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82592.log.gz 2026-02-20T21:47:15.823 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79891.log 2026-02-20T21:47:15.823 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57304.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57304.log.gz 2026-02-20T21:47:15.824 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129139.log 2026-02-20T21:47:15.824 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79891.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79891.log.gz 2026-02-20T21:47:15.824 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55073.log 2026-02-20T21:47:15.824 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129139.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.129139.log.gz 2026-02-20T21:47:15.824 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105989.log 2026-02-20T21:47:15.824 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130767.log.gz 2026-02-20T21:47:15.824 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55073.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.55073.log.gz 2026-02-20T21:47:15.824 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87505.log 2026-02-20T21:47:15.824 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105989.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105989.log.gz 2026-02-20T21:47:15.824 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142683.log 2026-02-20T21:47:15.828 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.87505.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.87505.log.gz 2026-02-20T21:47:15.828 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95582.log 2026-02-20T21:47:15.828 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142683.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.142683.log.gz 2026-02-20T21:47:15.828 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91612.log 2026-02-20T21:47:15.828 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95582.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95582.log.gz 2026-02-20T21:47:15.829 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74058.log 2026-02-20T21:47:15.832 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91612.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.94360.log.gz 2026-02-20T21:47:15.834 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138589.log 2026-02-20T21:47:15.834 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74058.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.74058.log.gz 2026-02-20T21:47:15.835 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90441.log 2026-02-20T21:47:15.835 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138589.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138589.log.gz 2026-02-20T21:47:15.835 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49525.log 2026-02-20T21:47:15.844 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90441.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.95251.log.gz 2026-02-20T21:47:15.853 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119647.log 2026-02-20T21:47:15.853 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.141949.log.gz 2026-02-20T21:47:15.853 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49525.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59341.log 2026-02-20T21:47:15.854 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49525.log.gz 2026-02-20T21:47:15.854 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119647.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72829.log 2026-02-20T21:47:15.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59341.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59341.log.gz 2026-02-20T21:47:15.855 INFO:teuthology.orchestra.run.trial163.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.119647.log.gz 2026-02-20T21:47:15.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122900.log 2026-02-20T21:47:15.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72829.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72829.log.gz 2026-02-20T21:47:15.856 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77601.log 2026-02-20T21:47:15.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122900.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122900.log.gz 2026-02-20T21:47:15.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86457.log 2026-02-20T21:47:15.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77601.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122203.log 2026-02-20T21:47:15.858 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77601.log.gz 2026-02-20T21:47:15.858 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86457.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141496.log 2026-02-20T21:47:15.858 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122203.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86457.log.gz 2026-02-20T21:47:15.859 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122203.log.gz 2026-02-20T21:47:15.859 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.142007.log 2026-02-20T21:47:15.859 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88489.log 2026-02-20T21:47:15.860 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141496.log: /var/log/ceph/cluster1-client.mirror.142007.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.141496.log.gz 2026-02-20T21:47:15.860 INFO:teuthology.orchestra.run.trial163.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.142007.log.gz 2026-02-20T21:47:15.860 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135251.log 2026-02-20T21:47:15.860 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93943.log 2026-02-20T21:47:15.861 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88489.log: /var/log/ceph/cluster2-client.mirror.135251.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.135251.log.gz 2026-02-20T21:47:15.862 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77566.log 2026-02-20T21:47:15.864 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93943.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88489.log.gz 2026-02-20T21:47:15.865 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83336.log 2026-02-20T21:47:15.865 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77566.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77566.log.gz 2026-02-20T21:47:15.866 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117572.log 2026-02-20T21:47:15.866 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83336.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83336.log.gz 2026-02-20T21:47:15.867 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136281.log 2026-02-20T21:47:15.867 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117572.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117572.log.gz 2026-02-20T21:47:15.868 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128445.log 2026-02-20T21:47:15.869 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136281.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.136281.log.gz 2026-02-20T21:47:15.869 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102680.log 2026-02-20T21:47:15.870 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128445.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.128445.log.gz 2026-02-20T21:47:15.870 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139666.log 2026-02-20T21:47:15.871 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102680.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93295.log.gz 2026-02-20T21:47:15.871 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.102680.log.gz 2026-02-20T21:47:15.874 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.117917.log.gz 2026-02-20T21:47:15.879 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124957.log 2026-02-20T21:47:15.880 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139666.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84876.log 2026-02-20T21:47:15.880 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124957.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135348.log 2026-02-20T21:47:15.880 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124957.log.gz 2026-02-20T21:47:15.881 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84876.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88383.log 2026-02-20T21:47:15.882 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135348.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.84876.log.gz 2026-02-20T21:47:15.882 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.135348.log.gz 2026-02-20T21:47:15.882 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64517.log 2026-02-20T21:47:15.882 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91885.log 2026-02-20T21:47:15.882 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88383.log: /var/log/ceph/cluster2-client.mirror.64517.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64517.log.gz 2026-02-20T21:47:15.883 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42851.log 2026-02-20T21:47:15.883 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91885.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91885.log.gz 2026-02-20T21:47:15.885 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130678.log 2026-02-20T21:47:15.885 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.42851.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.42851.log.gz 2026-02-20T21:47:15.885 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68834.log 2026-02-20T21:47:15.885 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130678.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.130678.log.gz 2026-02-20T21:47:15.885 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62599.log 2026-02-20T21:47:15.886 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68834.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68834.log.gz 2026-02-20T21:47:15.886 INFO:teuthology.orchestra.run.trial163.stderr:gzip 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88383.log.gz 2026-02-20T21:47:15.886 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125495.log 2026-02-20T21:47:15.887 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62599.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62599.log.gz 2026-02-20T21:47:15.887 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73143.log 2026-02-20T21:47:15.888 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125495.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.93345.log 2026-02-20T21:47:15.888 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73143.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73143.log.gz 2026-02-20T21:47:15.889 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113139.log 2026-02-20T21:47:15.898 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.93345.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73923.log 2026-02-20T21:47:15.901 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.113139.log: 92.1% 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.92203.log.gz 2026-02-20T21:47:15.901 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136139.log 2026-02-20T21:47:15.902 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73923.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73923.log.gz 2026-02-20T21:47:15.903 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91807.log 2026-02-20T21:47:15.903 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136139.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.136139.log.gz 2026-02-20T21:47:15.904 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87098.log 2026-02-20T21:47:15.904 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91807.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.91807.log.gz 2026-02-20T21:47:15.905 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75386.log 2026-02-20T21:47:15.906 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87098.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87098.log.gz 2026-02-20T21:47:15.906 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120052.log 2026-02-20T21:47:15.906 INFO:teuthology.orchestra.run.trial163.stderr: 95.6%/var/log/ceph/cluster2-client.mirror.75386.log: -- replaced with /var/log/ceph/cluster2-client.mirror.94668.log.gz 2026-02-20T21:47:15.906 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.113139.log.gz 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75386.log.gz 2026-02-20T21:47:15.906 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:15.906 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79333.log 2026-02-20T21:47:15.907 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53142.log 2026-02-20T21:47:15.907 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120052.log: /var/log/ceph/cluster2-client.mirror.79333.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120052.log.gz 2026-02-20T21:47:15.908 INFO:teuthology.orchestra.run.trial163.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.93345.log.gz 2026-02-20T21:47:15.908 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79333.log.gz 2026-02-20T21:47:15.909 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83429.log 2026-02-20T21:47:15.909 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53142.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104263.log 2026-02-20T21:47:15.909 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83429.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62878.log 2026-02-20T21:47:15.909 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.83429.log.gz 2026-02-20T21:47:15.910 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104263.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76327.log 2026-02-20T21:47:15.910 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62878.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104263.log.gz 2026-02-20T21:47:15.910 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62878.log.gz 2026-02-20T21:47:15.910 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51501.log 2026-02-20T21:47:15.910 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101407.log 2026-02-20T21:47:15.911 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76327.log: /var/log/ceph/cluster2-client.mirror.51501.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.76327.log.gz 2026-02-20T21:47:15.912 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51501.log.gz 2026-02-20T21:47:15.912 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63959.log 2026-02-20T21:47:15.912 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.101407.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137419.log 2026-02-20T21:47:15.913 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.101407.log.gz 2026-02-20T21:47:15.913 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63959.log: 88.6%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65126.log 2026-02-20T21:47:15.913 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.63959.log.gz 2026-02-20T21:47:15.913 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137419.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137419.log.gz 2026-02-20T21:47:15.913 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46092.log 2026-02-20T21:47:15.916 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109595.log 2026-02-20T21:47:15.916 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.65126.log: /var/log/ceph/cluster2-client.admin.46092.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46092.log.gz 2026-02-20T21:47:15.916 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.65126.log.gz 2026-02-20T21:47:15.916 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.138828.log.gz 2026-02-20T21:47:15.916 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74533.log 2026-02-20T21:47:15.916 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109595.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.101452.log 2026-02-20T21:47:15.916 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109595.log.gz 2026-02-20T21:47:15.916 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.74533.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54073.log 2026-02-20T21:47:15.916 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.admin.74533.log.gz 2026-02-20T21:47:15.916 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131782.log 2026-02-20T21:47:15.916 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.101452.log: gzip/var/log/ceph/cluster2-client.mirror.54073.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68776.log 2026-02-20T21:47:15.917 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131782.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131782.log.gz 2026-02-20T21:47:15.917 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.101452.log.gz 2026-02-20T21:47:15.917 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59279.log 2026-02-20T21:47:15.917 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133420.log 2026-02-20T21:47:15.918 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68776.log: /var/log/ceph/cluster2-client.mirror.59279.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68776.log.gz 2026-02-20T21:47:15.918 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.59279.log.gz 2026-02-20T21:47:15.918 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93345.log 2026-02-20T21:47:15.918 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133420.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.133420.log.gz 2026-02-20T21:47:15.919 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122773.log 2026-02-20T21:47:15.919 INFO:teuthology.orchestra.run.trial163.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.54073.log.gz 2026-02-20T21:47:15.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.93345.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58565.log 2026-02-20T21:47:15.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122773.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.47423.log 2026-02-20T21:47:15.920 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.122773.log.gz 2026-02-20T21:47:15.921 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58565.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58468.log 2026-02-20T21:47:15.921 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58565.log.gz 2026-02-20T21:47:15.921 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.3.47423.log: 94.8% -- replaced with /var/log/ceph/cluster1-client.mirror.93345.log.gz 2026-02-20T21:47:15.922 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123423.log 2026-02-20T21:47:15.922 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58468.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107786.log 2026-02-20T21:47:15.923 INFO:teuthology.orchestra.run.trial163.stderr:gzip/var/log/ceph/cluster2-client.mirror.123423.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69768.log 2026-02-20T21:47:15.923 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107786.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123423.log.gz 2026-02-20T21:47:15.923 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107786.log.gz 2026-02-20T21:47:15.923 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73294.log 2026-02-20T21:47:15.924 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69768.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69768.log.gz 2026-02-20T21:47:15.924 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100984.log 2026-02-20T21:47:15.925 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73294.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.142274.log 2026-02-20T21:47:15.925 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100984.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73294.log.gz 2026-02-20T21:47:15.925 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100984.log.gz 2026-02-20T21:47:15.925 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.96748.log 2026-02-20T21:47:15.926 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.142274.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.142274.log.gz 2026-02-20T21:47:15.926 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76250.log 2026-02-20T21:47:15.927 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93244.log 2026-02-20T21:47:15.927 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76250.log: /var/log/ceph/cluster2-client.mirror.96748.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.96748.log.gz 2026-02-20T21:47:15.927 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76250.log.gz 2026-02-20T21:47:15.927 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82685.log 2026-02-20T21:47:15.933 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93244.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.58468.log.gz 2026-02-20T21:47:15.937 INFO:teuthology.orchestra.run.trial163.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.53142.log.gz 2026-02-20T21:47:15.938 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76774.log.gz 2026-02-20T21:47:15.940 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138139.log 2026-02-20T21:47:15.940 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82685.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134898.log 2026-02-20T21:47:15.940 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82685.log.gz 2026-02-20T21:47:15.946 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138139.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136474.log 2026-02-20T21:47:15.946 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134898.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.138139.log.gz 2026-02-20T21:47:15.946 INFO:teuthology.orchestra.run.trial163.stderr:gzip 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.134898.log.gz 2026-02-20T21:47:15.946 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.admin.143604.log 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose --/var/log/ceph/cluster1-client.mirror.136474.log: /var/log/ceph/cluster1-client.mirror.1.47173.log 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr: 86.8%/var/log/ceph/cluster2-client.admin.143604.log: -- replaced with /var/log/ceph/cluster1-client.mirror.136474.log.gz 0.0% 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.143604.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109861.log 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93272.log 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.1.47173.log: /var/log/ceph/cluster1-client.mirror.109861.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.109861.log.gz 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109684.log 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93272.log: gzip 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93272.log.gz 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139618.log 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.102418.log.gz 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123558.log 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109684.log: /var/log/ceph/cluster2-client.mirror.139618.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87893.log 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.109684.log.gz 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr: 88.0%/var/log/ceph/cluster2-client.mirror.123558.log: -- replaced with /var/log/ceph/cluster2-client.mirror.139618.log.gz 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138364.log 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87893.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.123558.log.gz 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59438.log 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87893.log.gz 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138364.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90769.log 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.138364.log.gz 2026-02-20T21:47:15.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.59438.log: -- /var/log/ceph/cluster2-client.mirror.95997.log 2026-02-20T21:47:15.948 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59438.log.gz 2026-02-20T21:47:15.948 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79147.log 2026-02-20T21:47:15.948 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90769.log: /var/log/ceph/cluster2-client.mirror.95997.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122819.log 2026-02-20T21:47:15.948 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.95997.log.gz 2026-02-20T21:47:15.951 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79147.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79147.log.gz 2026-02-20T21:47:15.951 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107016.log 2026-02-20T21:47:15.951 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.93021.log.gz 2026-02-20T21:47:15.951 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122819.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141474.log 2026-02-20T21:47:15.951 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122819.log.gz 2026-02-20T21:47:15.951 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107016.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141258.log 2026-02-20T21:47:15.951 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63773.log 2026-02-20T21:47:15.951 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141474.log: /var/log/ceph/cluster2-client.mirror.141258.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.141258.log.gz 2026-02-20T21:47:15.951 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.141474.log.gz 2026-02-20T21:47:15.951 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65510.log.gz 2026-02-20T21:47:15.952 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83737.log 2026-02-20T21:47:15.952 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63773.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136677.log 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63773.log.gz 2026-02-20T21:47:15.952 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:15.952 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83737.log: gzip 89.0% -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127948.log 2026-02-20T21:47:15.952 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.83737.log.gz 2026-02-20T21:47:15.952 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136677.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124525.log 2026-02-20T21:47:15.952 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.136677.log.gz 2026-02-20T21:47:15.952 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63215.log 2026-02-20T21:47:15.953 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127948.log: /var/log/ceph/cluster2-client.mirror.124525.log: 88.2% 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124525.log.gz 2026-02-20T21:47:15.953 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.127948.log.gz 2026-02-20T21:47:15.953 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58759.log 2026-02-20T21:47:15.953 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63215.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63215.log.gz 2026-02-20T21:47:15.953 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92998.log 2026-02-20T21:47:15.953 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58759.log: gzip 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58759.log.gz 2026-02-20T21:47:15.953 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109100.log 2026-02-20T21:47:15.954 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92998.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.92998.log.gz 2026-02-20T21:47:15.957 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109100.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106442.log 2026-02-20T21:47:15.957 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72581.log 2026-02-20T21:47:15.958 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106442.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85245.log 2026-02-20T21:47:15.958 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72581.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72581.log.gz 2026-02-20T21:47:15.968 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142209.log 2026-02-20T21:47:15.968 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85245.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.85245.log.gz 2026-02-20T21:47:15.969 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76292.log 2026-02-20T21:47:15.971 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142209.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.142209.log.gz 2026-02-20T21:47:15.971 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125315.log 2026-02-20T21:47:15.972 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76292.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.76292.log.gz 2026-02-20T21:47:15.972 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105064.log 2026-02-20T21:47:15.973 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125315.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125315.log.gz 2026-02-20T21:47:15.974 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57595.log 2026-02-20T21:47:15.974 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105064.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.105064.log.gz 2026-02-20T21:47:15.975 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133462.log 2026-02-20T21:47:15.975 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100029.log 2026-02-20T21:47:15.976 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133462.log: /var/log/ceph/cluster2-client.mirror.57595.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.133462.log.gz 2026-02-20T21:47:15.979 INFO:teuthology.orchestra.run.trial163.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.3.47423.log.gz 2026-02-20T21:47:15.979 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.57595.log.gz 2026-02-20T21:47:15.981 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118309.log 2026-02-20T21:47:15.982 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141015.log 2026-02-20T21:47:15.982 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100029.log: /var/log/ceph/cluster2-client.mirror.118309.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100029.log.gz 2026-02-20T21:47:15.983 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118309.log.gz 2026-02-20T21:47:15.983 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69485.log 2026-02-20T21:47:15.983 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141015.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124317.log 2026-02-20T21:47:15.983 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.141015.log.gz/var/log/ceph/cluster2-client.mirror.69485.log: 2026-02-20T21:47:15.983 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69485.log.gz 2026-02-20T21:47:15.985 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.92283.log.gz 2026-02-20T21:47:15.994 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73108.log 2026-02-20T21:47:15.996 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124317.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54522.log 2026-02-20T21:47:15.996 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73108.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124317.log.gz 2026-02-20T21:47:15.996 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73712.log 2026-02-20T21:47:15.996 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54522.log: 88.8%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.73108.log.gz 2026-02-20T21:47:15.996 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83829.log 2026-02-20T21:47:15.996 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.54522.log.gz 2026-02-20T21:47:15.996 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60797.log 2026-02-20T21:47:15.996 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.73712.log: /var/log/ceph/cluster2-client.mirror.83829.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135447.log 2026-02-20T21:47:15.997 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60797.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60797.log.gz 2026-02-20T21:47:15.997 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76950.log 2026-02-20T21:47:15.999 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135447.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.135447.log.gz 2026-02-20T21:47:15.999 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122091.log 2026-02-20T21:47:16.000 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73712.log.gz 2026-02-20T21:47:16.000 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76950.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89329.log 2026-02-20T21:47:16.000 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.76950.log.gz 2026-02-20T21:47:16.001 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122091.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62785.log 2026-02-20T21:47:16.001 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122091.log.gz 2026-02-20T21:47:16.001 INFO:teuthology.orchestra.run.trial163.stderr: 93.6%/var/log/ceph/cluster2-client.mirror.89329.log: -- replaced with /var/log/ceph/cluster2-client.mirror.83829.log.gz 2026-02-20T21:47:16.001 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59632.log 2026-02-20T21:47:16.003 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62785.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62785.log.gz 2026-02-20T21:47:16.003 INFO:teuthology.orchestra.run.trial163.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89329.log.gz 2026-02-20T21:47:16.003 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130019.log 2026-02-20T21:47:16.003 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59632.log: gzip -5 --verbose -- 88.5% /var/log/ceph/cluster2-client.mirror.104802.log 2026-02-20T21:47:16.004 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.59632.log.gz 2026-02-20T21:47:16.004 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130019.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97460.log 2026-02-20T21:47:16.004 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.130019.log.gz 2026-02-20T21:47:16.004 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106158.log 2026-02-20T21:47:16.004 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104802.log: /var/log/ceph/cluster2-client.mirror.97460.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80112.log 2026-02-20T21:47:16.004 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97460.log.gz 2026-02-20T21:47:16.005 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106158.log: gzip 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106158.log.gz 2026-02-20T21:47:16.005 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124398.log 2026-02-20T21:47:16.005 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80112.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.80112.log.gz 2026-02-20T21:47:16.005 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96991.log 2026-02-20T21:47:16.005 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70733.log 2026-02-20T21:47:16.006 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124398.log: /var/log/ceph/cluster2-client.mirror.96991.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124398.log.gz 2026-02-20T21:47:16.006 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.96991.log.gz 2026-02-20T21:47:16.006 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69803.log 2026-02-20T21:47:16.007 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70733.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76546.log 2026-02-20T21:47:16.007 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70733.log.gz 2026-02-20T21:47:16.007 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69803.log: 88.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68683.log 2026-02-20T21:47:16.007 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.69803.log.gz 2026-02-20T21:47:16.012 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76546.log: 96.8% -- replaced with /var/log/ceph/cluster2-client.mirror.76546.log.gz 2026-02-20T21:47:16.012 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65558.log 2026-02-20T21:47:16.013 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68683.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68683.log.gz 2026-02-20T21:47:16.014 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92622.log 2026-02-20T21:47:16.014 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65558.log: 76.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65558.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132157.log 2026-02-20T21:47:16.014 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:16.015 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92622.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.142062.log 2026-02-20T21:47:16.015 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132157.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.92622.log.gz 2026-02-20T21:47:16.015 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132157.log.gz 2026-02-20T21:47:16.015 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117694.log 2026-02-20T21:47:16.016 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.142062.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.142062.log.gz 2026-02-20T21:47:16.016 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118182.log 2026-02-20T21:47:16.016 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119860.log 2026-02-20T21:47:16.016 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%/var/log/ceph/cluster2-client.mirror.118182.log: -- replaced with /var/log/ceph/cluster2-client.mirror.117694.log.gz 2026-02-20T21:47:16.016 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118182.log.gz 2026-02-20T21:47:16.017 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50552.log 2026-02-20T21:47:16.017 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119860.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119860.log.gz 2026-02-20T21:47:16.017 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101085.log 2026-02-20T21:47:16.018 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50552.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110007.log 2026-02-20T21:47:16.020 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50552.log.gz 2026-02-20T21:47:16.020 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101085.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132525.log 2026-02-20T21:47:16.020 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.101085.log.gz 2026-02-20T21:47:16.020 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110007.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97080.log 2026-02-20T21:47:16.020 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.110007.log.gz 2026-02-20T21:47:16.020 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132525.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107523.log 2026-02-20T21:47:16.020 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132525.log.gz 2026-02-20T21:47:16.020 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97172.log 2026-02-20T21:47:16.022 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97080.log: /var/log/ceph/cluster1-client.mirror.107523.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97080.log.gz 2026-02-20T21:47:16.022 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.107523.log.gz 2026-02-20T21:47:16.022 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73329.log 2026-02-20T21:47:16.022 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140973.log 2026-02-20T21:47:16.022 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73329.log: /var/log/ceph/cluster2-client.mirror.97172.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73329.log.gz 2026-02-20T21:47:16.022 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.97172.log.gz 2026-02-20T21:47:16.023 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135131.log 2026-02-20T21:47:16.024 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137689.log 2026-02-20T21:47:16.024 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135131.log: /var/log/ceph/cluster1-client.mirror.140973.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.140973.log.gz 2026-02-20T21:47:16.024 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.135131.log.gz 2026-02-20T21:47:16.026 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127602.log 2026-02-20T21:47:16.027 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57207.log 2026-02-20T21:47:16.027 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127602.log: /var/log/ceph/cluster1-client.mirror.137689.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137689.log.gz 2026-02-20T21:47:16.027 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124876.log 2026-02-20T21:47:16.028 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57207.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57207.log.gz 2026-02-20T21:47:16.028 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51606.log 2026-02-20T21:47:16.029 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124876.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124876.log.gz 2026-02-20T21:47:16.029 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65214.log 2026-02-20T21:47:16.030 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51606.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51606.log.gz 2026-02-20T21:47:16.030 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66509.log 2026-02-20T21:47:16.034 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.65214.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.65214.log.gz 2026-02-20T21:47:16.034 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79833.log 2026-02-20T21:47:16.035 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66509.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66509.log.gz 2026-02-20T21:47:16.035 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136352.log 2026-02-20T21:47:16.036 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79833.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79833.log.gz 2026-02-20T21:47:16.036 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85144.log 2026-02-20T21:47:16.037 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136352.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.136352.log.gz 2026-02-20T21:47:16.038 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54328.log 2026-02-20T21:47:16.038 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.103081.log.gz 2026-02-20T21:47:16.038 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85144.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.85144.log.gz 2026-02-20T21:47:16.038 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106247.log 2026-02-20T21:47:16.040 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79519.log 2026-02-20T21:47:16.040 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106247.log: /var/log/ceph/cluster2-client.mirror.54328.log: 88.2% 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54328.log.gz 2026-02-20T21:47:16.040 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.106247.log.gz 2026-02-20T21:47:16.040 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115578.log 2026-02-20T21:47:16.040 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79519.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137307.log 2026-02-20T21:47:16.040 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79519.log.gz 2026-02-20T21:47:16.043 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115578.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89417.log 2026-02-20T21:47:16.043 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.115578.log.gz 2026-02-20T21:47:16.043 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137307.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.142729.log 2026-02-20T21:47:16.043 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89417.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137307.log.gz 2026-02-20T21:47:16.043 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129514.log 2026-02-20T21:47:16.043 INFO:teuthology.orchestra.run.trial163.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89417.log.gz 2026-02-20T21:47:16.043 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136587.log 2026-02-20T21:47:16.043 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.142729.log: /var/log/ceph/cluster2-client.mirror.129514.log: 28.1% -- replaced with /var/log/ceph/cluster1-client.admin.142729.log.gz 2026-02-20T21:47:16.043 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.129514.log.gz 2026-02-20T21:47:16.043 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139558.log 2026-02-20T21:47:16.043 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136587.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121313.log 2026-02-20T21:47:16.043 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136587.log.gz 2026-02-20T21:47:16.046 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139558.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137104.log 2026-02-20T21:47:16.046 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.139558.log.gz 2026-02-20T21:47:16.046 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121313.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61297.log 2026-02-20T21:47:16.046 INFO:teuthology.orchestra.run.trial163.stderr: 87.9%/var/log/ceph/cluster1-client.mirror.137104.log: -- replaced with /var/log/ceph/cluster2-client.mirror.121313.log.gz 2026-02-20T21:47:16.046 INFO:teuthology.orchestra.run.trial163.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.137104.log.gz 2026-02-20T21:47:16.046 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109794.log 2026-02-20T21:47:16.046 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61297.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128639.log 2026-02-20T21:47:16.046 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.61297.log.gz 2026-02-20T21:47:16.046 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109794.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106615.log 2026-02-20T21:47:16.046 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.109794.log.gz 2026-02-20T21:47:16.046 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128639.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87164.log 2026-02-20T21:47:16.046 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106615.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128639.log.gz 2026-02-20T21:47:16.048 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106615.log.gz 2026-02-20T21:47:16.048 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76404.log 2026-02-20T21:47:16.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87164.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83541.log 2026-02-20T21:47:16.048 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87164.log.gz 2026-02-20T21:47:16.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76404.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124282.log 2026-02-20T21:47:16.048 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76404.log.gz 2026-02-20T21:47:16.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83541.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130064.log 2026-02-20T21:47:16.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124282.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.83541.log.gz 2026-02-20T21:47:16.049 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124282.log.gz 2026-02-20T21:47:16.049 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-20T21:47:16.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130064.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115720.log 2026-02-20T21:47:16.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65017.log 2026-02-20T21:47:16.049 INFO:teuthology.orchestra.run.trial163.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.130064.log.gz 2026-02-20T21:47:16.051 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115720.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.115720.log.gz 2026-02-20T21:47:16.051 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78124.log 2026-02-20T21:47:16.052 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65017.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65017.log.gz 2026-02-20T21:47:16.052 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126368.log 2026-02-20T21:47:16.053 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78124.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78124.log.gz 2026-02-20T21:47:16.053 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69206.log 2026-02-20T21:47:16.062 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126368.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139599.log 2026-02-20T21:47:16.062 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69206.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69206.log.gz 2026-02-20T21:47:16.066 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.109100.log.gz 2026-02-20T21:47:16.067 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139408.log 2026-02-20T21:47:16.067 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139599.log: 83.4% -- replaced with /var/log/ceph/cluster2-client.mirror.139599.log.gz 2026-02-20T21:47:16.080 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101120.log 2026-02-20T21:47:16.081 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139408.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.139408.log.gz 2026-02-20T21:47:16.103 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51711.log 2026-02-20T21:47:16.104 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101120.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.101120.log.gz 2026-02-20T21:47:16.104 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89915.log 2026-02-20T21:47:16.104 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51711.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51711.log.gz 2026-02-20T21:47:16.104 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71384.log 2026-02-20T21:47:16.104 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89915.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.89915.log.gz 2026-02-20T21:47:16.104 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133229.log 2026-02-20T21:47:16.104 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71384.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71384.log.gz 2026-02-20T21:47:16.104 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141980.log 2026-02-20T21:47:16.104 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133229.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133229.log.gz 2026-02-20T21:47:16.104 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54914.log 2026-02-20T21:47:16.105 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.141980.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.141980.log.gz 2026-02-20T21:47:16.105 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91830.log 2026-02-20T21:47:16.105 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54914.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54914.log.gz 2026-02-20T21:47:16.105 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90986.log 2026-02-20T21:47:16.111 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91830.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108853.log 2026-02-20T21:47:16.123 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90986.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.90986.log.gz 2026-02-20T21:47:16.124 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139302.log 2026-02-20T21:47:16.125 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108853.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.108853.log.gz 2026-02-20T21:47:16.128 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.92933.log.gz 2026-02-20T21:47:16.130 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.110674.log 2026-02-20T21:47:16.130 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139302.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.139302.log.gz 2026-02-20T21:47:16.130 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117818.log 2026-02-20T21:47:16.136 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.110674.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.110674.log.gz 2026-02-20T21:47:16.136 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72643.log 2026-02-20T21:47:16.136 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117818.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117818.log.gz 2026-02-20T21:47:16.136 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43025.log 2026-02-20T21:47:16.136 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72643.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72643.log.gz 2026-02-20T21:47:16.136 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49595.log 2026-02-20T21:47:16.136 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.43025.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43025.log.gz 2026-02-20T21:47:16.136 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106917.log 2026-02-20T21:47:16.136 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49595.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60567.log 2026-02-20T21:47:16.136 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49595.log.gz 2026-02-20T21:47:16.136 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106917.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.106917.log.gz 2026-02-20T21:47:16.137 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113161.log 2026-02-20T21:47:16.148 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60567.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74170.log 2026-02-20T21:47:16.158 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113161.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129987.log 2026-02-20T21:47:16.159 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74170.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74170.log.gz 2026-02-20T21:47:16.182 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.104802.log.gz 2026-02-20T21:47:16.183 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134663.log 2026-02-20T21:47:16.184 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129987.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.129987.log.gz 2026-02-20T21:47:16.196 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95702.log 2026-02-20T21:47:16.197 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134663.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134663.log.gz 2026-02-20T21:47:16.205 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58115.log 2026-02-20T21:47:16.218 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95702.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.95702.log.gz 2026-02-20T21:47:16.219 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114959.log 2026-02-20T21:47:16.220 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58115.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58115.log.gz 2026-02-20T21:47:16.228 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58794.log 2026-02-20T21:47:16.236 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114959.log: 88.3% 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.139666.log.gz 2026-02-20T21:47:16.237 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77473.log 2026-02-20T21:47:16.237 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58794.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58794.log.gz 2026-02-20T21:47:16.237 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.76112.log 2026-02-20T21:47:16.237 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77473.log: -- replaced with /var/log/ceph/cluster2-client.mirror.114959.log.gz 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77473.log.gz 2026-02-20T21:47:16.237 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:16.237 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133549.log 2026-02-20T21:47:16.240 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.1.76112.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.1.76112.log.gz 2026-02-20T21:47:16.240 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84635.log 2026-02-20T21:47:16.240 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133549.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.133549.log.gz 2026-02-20T21:47:16.240 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113027.log 2026-02-20T21:47:16.240 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84635.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.84635.log.gz 2026-02-20T21:47:16.245 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99305.log 2026-02-20T21:47:16.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113027.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113027.log.gz 2026-02-20T21:47:16.245 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75979.log 2026-02-20T21:47:16.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99305.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99305.log.gz 2026-02-20T21:47:16.245 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61262.log 2026-02-20T21:47:16.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75979.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75979.log.gz 2026-02-20T21:47:16.245 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.119448.log 2026-02-20T21:47:16.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61262.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61262.log.gz 2026-02-20T21:47:16.245 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139100.log 2026-02-20T21:47:16.248 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.119448.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.127602.log.gz 2026-02-20T21:47:16.249 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117188.log 2026-02-20T21:47:16.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139100.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.139100.log.gz 2026-02-20T21:47:16.250 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136609.log 2026-02-20T21:47:16.259 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117188.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117188.log.gz 2026-02-20T21:47:16.259 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67532.log 2026-02-20T21:47:16.259 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136609.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136609.log.gz 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80914.log 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67532.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.125495.log.gz 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67532.log.gz 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109078.log 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80914.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84468.log 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80914.log.gz 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.109078.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131493.log 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84468.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.84468.log.gz 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.109078.log.gz 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114797.log 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131493.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78310.log 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131493.log.gz 2026-02-20T21:47:16.260 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114797.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114797.log.gz 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131879.log 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78310.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82309.log 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78310.log.gz 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131879.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.131879.log.gz 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112343.log 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82309.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49700.log 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82309.log.gz 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112343.log: 87.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64087.log 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.112343.log.gz 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49700.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54093.log 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%/var/log/ceph/cluster2-client.mirror.64087.log: -- replaced with /var/log/ceph/cluster2-client.mirror.49700.log.gz 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.64087.log.gz 2026-02-20T21:47:16.261 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88641.log 2026-02-20T21:47:16.262 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.54093.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94022.log 2026-02-20T21:47:16.265 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88641.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.54093.log.gz 2026-02-20T21:47:16.265 INFO:teuthology.orchestra.run.trial163.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88641.log.gz 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115695.log 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94022.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.45844.log 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.94022.log.gz 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.115695.log: gzip -5 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.115695.log.gz 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.57339.log 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.45844.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.45844.log.gz 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126055.log 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57339.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124779.log 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr: 88.5%/var/log/ceph/cluster2-client.mirror.126055.log: -- replaced with /var/log/ceph/cluster2-client.mirror.57339.log.gz 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135498.log 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124779.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124779.log.gz 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126055.log.gz 2026-02-20T21:47:16.266 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53448.log 2026-02-20T21:47:16.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135498.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51536.log 2026-02-20T21:47:16.271 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53448.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.135498.log.gz 2026-02-20T21:47:16.271 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53448.log.gz 2026-02-20T21:47:16.271 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56078.log 2026-02-20T21:47:16.271 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51536.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69989.log 2026-02-20T21:47:16.271 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56078.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.51536.log.gz 2026-02-20T21:47:16.271 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56078.log.gz 2026-02-20T21:47:16.271 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120087.log 2026-02-20T21:47:16.271 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69989.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130933.log 2026-02-20T21:47:16.271 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%/var/log/ceph/cluster2-client.mirror.120087.log: -- replaced with /var/log/ceph/cluster2-client.mirror.69989.log.gz 2026-02-20T21:47:16.271 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120087.log.gz 2026-02-20T21:47:16.271 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142187.log 2026-02-20T21:47:16.271 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130933.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101617.log 2026-02-20T21:47:16.272 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142187.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.142187.log.gz 2026-02-20T21:47:16.272 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50762.log 2026-02-20T21:47:16.272 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.130933.log.gz/var/log/ceph/cluster2-client.mirror.101617.log: 2026-02-20T21:47:16.272 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101617.log.gz 2026-02-20T21:47:16.272 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.98020.log 2026-02-20T21:47:16.272 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50762.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123481.log 2026-02-20T21:47:16.273 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.3.98020.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50762.log.gz 2026-02-20T21:47:16.274 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105242.log 2026-02-20T21:47:16.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123481.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.123481.log.gz 2026-02-20T21:47:16.274 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83872.log 2026-02-20T21:47:16.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105242.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105242.log.gz 2026-02-20T21:47:16.274 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72080.log 2026-02-20T21:47:16.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83872.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.0.119448.log.gz 2026-02-20T21:47:16.275 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101718.log 2026-02-20T21:47:16.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72080.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72080.log.gz 2026-02-20T21:47:16.276 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82813.log 2026-02-20T21:47:16.276 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101718.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101718.log.gz 2026-02-20T21:47:16.279 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119622.log 2026-02-20T21:47:16.279 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.83872.log.gz 2026-02-20T21:47:16.279 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82813.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100860.log 2026-02-20T21:47:16.279 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.119622.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82813.log.gz 2026-02-20T21:47:16.279 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.119622.log.gz 2026-02-20T21:47:16.279 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.142833.log 2026-02-20T21:47:16.279 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100860.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112779.log 2026-02-20T21:47:16.279 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.142833.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.142833.log.gz 2026-02-20T21:47:16.279 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93631.log 2026-02-20T21:47:16.280 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112779.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112779.log.gz 2026-02-20T21:47:16.280 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74478.log 2026-02-20T21:47:16.290 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93631.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.98020.log.gz 2026-02-20T21:47:16.291 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104905.log 2026-02-20T21:47:16.298 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74478.log: 97.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74478.log.gz 2026-02-20T21:47:16.299 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88028.log 2026-02-20T21:47:16.299 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104905.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104905.log.gz 2026-02-20T21:47:16.299 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139814.log 2026-02-20T21:47:16.299 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88028.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.88028.log.gz 2026-02-20T21:47:16.301 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121201.log 2026-02-20T21:47:16.301 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139814.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.139814.log.gz 2026-02-20T21:47:16.301 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127913.log 2026-02-20T21:47:16.301 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121201.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.121201.log.gz 2026-02-20T21:47:16.302 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90632.log 2026-02-20T21:47:16.302 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127913.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127913.log.gz 2026-02-20T21:47:16.302 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132365.log 2026-02-20T21:47:16.313 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90632.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.126368.log.gz 2026-02-20T21:47:16.314 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125453.log 2026-02-20T21:47:16.314 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132365.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132365.log.gz 2026-02-20T21:47:16.315 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136654.log 2026-02-20T21:47:16.316 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125453.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.125453.log.gz 2026-02-20T21:47:16.317 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139162.log 2026-02-20T21:47:16.317 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136654.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136654.log.gz 2026-02-20T21:47:16.318 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141434.log 2026-02-20T21:47:16.318 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139162.log: 83.4% -- replaced with /var/log/ceph/cluster2-client.mirror.139162.log.gz 2026-02-20T21:47:16.319 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128604.log 2026-02-20T21:47:16.320 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141434.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.141434.log.gz 2026-02-20T21:47:16.320 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135618.log 2026-02-20T21:47:16.322 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128604.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.128604.log.gz 2026-02-20T21:47:16.322 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.142235.log.gz 2026-02-20T21:47:16.322 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48430.log 2026-02-20T21:47:16.323 INFO:teuthology.orchestra.run.trial163.stderr:gzip/var/log/ceph/cluster2-client.mirror.135618.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111896.log 2026-02-20T21:47:16.323 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48430.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.135618.log.gz 2026-02-20T21:47:16.323 INFO:teuthology.orchestra.run.trial163.stderr: 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.48430.log.gz 2026-02-20T21:47:16.323 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123242.log 2026-02-20T21:47:16.325 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140788.log 2026-02-20T21:47:16.325 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111896.log: /var/log/ceph/cluster1-client.mirror.123242.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.111896.log.gz 2026-02-20T21:47:16.326 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53645.log 2026-02-20T21:47:16.326 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140788.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.123242.log.gz 2026-02-20T21:47:16.326 INFO:teuthology.orchestra.run.trial163.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.140788.log.gz 2026-02-20T21:47:16.326 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108481.log 2026-02-20T21:47:16.326 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53645.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139141.log 2026-02-20T21:47:16.326 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108481.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53645.log.gz 2026-02-20T21:47:16.329 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85649.log 2026-02-20T21:47:16.329 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.108481.log.gz 2026-02-20T21:47:16.329 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139141.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89938.log 2026-02-20T21:47:16.329 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85649.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.85649.log.gz 2026-02-20T21:47:16.329 INFO:teuthology.orchestra.run.trial163.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.139141.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133249.log 2026-02-20T21:47:16.329 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:16.329 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89938.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133953.log 2026-02-20T21:47:16.330 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133249.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.133249.log.gz 2026-02-20T21:47:16.334 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125428.log 2026-02-20T21:47:16.335 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133953.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.133953.log.gz 2026-02-20T21:47:16.337 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105776.log 2026-02-20T21:47:16.337 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.125428.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.125428.log.gz 2026-02-20T21:47:16.337 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83150.log 2026-02-20T21:47:16.337 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105776.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105776.log.gz 2026-02-20T21:47:16.338 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82402.log 2026-02-20T21:47:16.338 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83150.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.83150.log.gz 2026-02-20T21:47:16.340 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89396.log 2026-02-20T21:47:16.340 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82402.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82402.log.gz 2026-02-20T21:47:16.340 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137554.log 2026-02-20T21:47:16.342 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89396.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.89396.log.gz 2026-02-20T21:47:16.342 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103897.log 2026-02-20T21:47:16.343 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137554.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137554.log.gz 2026-02-20T21:47:16.343 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.91693.log 2026-02-20T21:47:16.343 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.93943.log.gz 2026-02-20T21:47:16.343 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103897.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86356.log 2026-02-20T21:47:16.351 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.2.91693.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140022.log 2026-02-20T21:47:16.364 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86356.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.86356.log.gz 2026-02-20T21:47:16.366 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107548.log 2026-02-20T21:47:16.367 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.140022.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.140022.log.gz 2026-02-20T21:47:16.379 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125151.log 2026-02-20T21:47:16.379 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107548.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.107548.log.gz 2026-02-20T21:47:16.386 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91009.log.gz 2026-02-20T21:47:16.387 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107590.log 2026-02-20T21:47:16.387 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125151.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.125151.log.gz 2026-02-20T21:47:16.395 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128275.log 2026-02-20T21:47:16.404 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107590.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51396.log 2026-02-20T21:47:16.424 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.128275.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.106442.log.gz 2026-02-20T21:47:16.424 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123593.log 2026-02-20T21:47:16.424 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51396.log: 88.1% 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51396.log.gz 2026-02-20T21:47:16.424 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.128275.log.gz 2026-02-20T21:47:16.424 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78403.log 2026-02-20T21:47:16.424 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123593.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.123593.log.gz 2026-02-20T21:47:16.425 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109972.log 2026-02-20T21:47:16.425 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78403.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78403.log.gz 2026-02-20T21:47:16.425 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124592.log 2026-02-20T21:47:16.425 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109972.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.109972.log.gz 2026-02-20T21:47:16.425 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87773.log 2026-02-20T21:47:16.425 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.124592.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.124592.log.gz 2026-02-20T21:47:16.425 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64982.log 2026-02-20T21:47:16.425 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87773.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87773.log.gz 2026-02-20T21:47:16.425 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131547.log 2026-02-20T21:47:16.426 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64982.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64982.log.gz 2026-02-20T21:47:16.426 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65171.log 2026-02-20T21:47:16.426 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131547.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131547.log.gz 2026-02-20T21:47:16.426 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114131.log 2026-02-20T21:47:16.437 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.65171.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.65171.log.gz 2026-02-20T21:47:16.437 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.119667.log 2026-02-20T21:47:16.437 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114131.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114131.log.gz 2026-02-20T21:47:16.437 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103580.log 2026-02-20T21:47:16.437 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.119667.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.119667.log.gz 2026-02-20T21:47:16.437 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117875.log 2026-02-20T21:47:16.437 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103580.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103580.log.gz 2026-02-20T21:47:16.437 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134876.log 2026-02-20T21:47:16.437 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117875.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.117875.log.gz 2026-02-20T21:47:16.438 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75293.log 2026-02-20T21:47:16.438 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134876.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.134876.log.gz 2026-02-20T21:47:16.438 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93244.log.gz 2026-02-20T21:47:16.438 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57533.log 2026-02-20T21:47:16.438 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75293.log: 89.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59147.log 2026-02-20T21:47:16.438 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.75293.log.gz 2026-02-20T21:47:16.438 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57533.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57533.log.gz 2026-02-20T21:47:16.438 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46238.log 2026-02-20T21:47:16.438 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75886.log 2026-02-20T21:47:16.438 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59147.log: /var/log/ceph/cluster2-client.admin.46238.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46238.log.gz 2026-02-20T21:47:16.438 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59147.log.gz 2026-02-20T21:47:16.439 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56237.log 2026-02-20T21:47:16.439 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53049.log 2026-02-20T21:47:16.439 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75886.log: /var/log/ceph/cluster2-client.mirror.56237.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75886.log.gz 2026-02-20T21:47:16.439 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56237.log.gz 2026-02-20T21:47:16.439 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58274.log 2026-02-20T21:47:16.439 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110654.log 2026-02-20T21:47:16.439 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53049.log: /var/log/ceph/cluster2-client.mirror.58274.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51431.log 2026-02-20T21:47:16.439 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58274.log.gz 2026-02-20T21:47:16.446 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.72322.log.gz 2026-02-20T21:47:16.449 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98667.log 2026-02-20T21:47:16.449 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51431.log: gzip/var/log/ceph/cluster2-client.mirror.110654.log: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131662.log 2026-02-20T21:47:16.449 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98667.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51431.log.gz 2026-02-20T21:47:16.449 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98667.log.gz 2026-02-20T21:47:16.449 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135876.log 2026-02-20T21:47:16.449 INFO:teuthology.orchestra.run.trial163.stderr: 92.1% -- replaced with /var/log/ceph/cluster2-client.mirror.110654.log.gz 2026-02-20T21:47:16.449 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.116566.log 2026-02-20T21:47:16.449 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135876.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.131662.log.gz 2026-02-20T21:47:16.450 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74921.log 2026-02-20T21:47:16.450 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.3.116566.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.135876.log.gz 2026-02-20T21:47:16.451 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63064.log 2026-02-20T21:47:16.451 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74921.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.74921.log.gz 2026-02-20T21:47:16.460 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83092.log 2026-02-20T21:47:16.461 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63064.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63064.log.gz 2026-02-20T21:47:16.461 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93970.log 2026-02-20T21:47:16.466 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83092.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.83092.log.gz 2026-02-20T21:47:16.466 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117305.log 2026-02-20T21:47:16.466 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93970.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.93970.log.gz 2026-02-20T21:47:16.467 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110142.log 2026-02-20T21:47:16.467 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117305.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117305.log.gz 2026-02-20T21:47:16.467 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103375.log 2026-02-20T21:47:16.467 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110142.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110142.log.gz 2026-02-20T21:47:16.467 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91233.log 2026-02-20T21:47:16.467 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103375.log: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.116566.log.gz 2026-02-20T21:47:16.467 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103375.log.gz 2026-02-20T21:47:16.467 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129344.log 2026-02-20T21:47:16.467 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104174.log 2026-02-20T21:47:16.479 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91233.log: /var/log/ceph/cluster2-client.mirror.129344.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.129344.log.gz 2026-02-20T21:47:16.479 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115740.log 2026-02-20T21:47:16.479 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104174.log: 95.7% 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107590.log.gz 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.104174.log.gz 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74501.log 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79461.log 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.115740.log: /var/log/ceph/cluster2-client.mirror.74501.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.115740.log.gz 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136767.log 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79461.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79461.log.gz 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130913.log 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136767.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136767.log.gz 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.37397.log 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130913.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.130913.log.gz 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73557.log 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/ceph.tmp-client.admin.37397.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.37397.log.gz 2026-02-20T21:47:16.480 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.138802.log 2026-02-20T21:47:16.481 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73557.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73557.log.gz 2026-02-20T21:47:16.481 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132869.log 2026-02-20T21:47:16.481 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.138802.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.138802.log.gz 2026-02-20T21:47:16.481 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137194.log 2026-02-20T21:47:16.481 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132869.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132869.log.gz 2026-02-20T21:47:16.481 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60298.log 2026-02-20T21:47:16.481 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137194.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.137194.log.gz 2026-02-20T21:47:16.481 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134308.log 2026-02-20T21:47:16.481 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60298.log: 77.3% -- replaced with /var/log/ceph/cluster2-client.mirror.60298.log.gz 2026-02-20T21:47:16.481 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132847.log 2026-02-20T21:47:16.481 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134308.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134308.log.gz 2026-02-20T21:47:16.481 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92564.log 2026-02-20T21:47:16.482 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132847.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132847.log.gz 2026-02-20T21:47:16.483 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138049.log 2026-02-20T21:47:16.487 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92564.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89869.log.gz 2026-02-20T21:47:16.489 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.89198.log 2026-02-20T21:47:16.489 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138049.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.138049.log.gz 2026-02-20T21:47:16.489 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54782.log 2026-02-20T21:47:16.489 INFO:teuthology.orchestra.run.trial163.stderr: 95.7%/var/log/ceph/cluster2-client.mirror.1.89198.log: -- replaced with /var/log/ceph/cluster2-client.mirror.52959.log.gz 2026-02-20T21:47:16.490 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.60298.log 2026-02-20T21:47:16.490 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54782.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.54782.log.gz 2026-02-20T21:47:16.501 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52624.log 2026-02-20T21:47:16.502 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.1.60298.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.60298.log.gz 2026-02-20T21:47:16.503 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53695.log 2026-02-20T21:47:16.505 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52624.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52624.log.gz 2026-02-20T21:47:16.505 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122935.log 2026-02-20T21:47:16.505 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53695.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141455.log 2026-02-20T21:47:16.505 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53695.log.gz 2026-02-20T21:47:16.507 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122935.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122935.log.gz 2026-02-20T21:47:16.507 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135368.log 2026-02-20T21:47:16.507 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141455.log: 83.3% -- replaced with /var/log/ceph/cluster2-client.mirror.141455.log.gz 2026-02-20T21:47:16.507 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72922.log 2026-02-20T21:47:16.508 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135368.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.135368.log.gz 2026-02-20T21:47:16.509 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64180.log 2026-02-20T21:47:16.510 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72922.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72922.log.gz 2026-02-20T21:47:16.510 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70884.log 2026-02-20T21:47:16.510 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64180.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64180.log.gz 2026-02-20T21:47:16.512 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51816.log 2026-02-20T21:47:16.512 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70884.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70884.log.gz 2026-02-20T21:47:16.512 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134350.log 2026-02-20T21:47:16.516 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51816.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51816.log.gz 2026-02-20T21:47:16.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64831.log 2026-02-20T21:47:16.517 INFO:teuthology.orchestra.run.trial163.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.1.89198.log.gz 2026-02-20T21:47:16.517 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134350.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-20T21:47:16.517 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64831.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64831.log.gz 2026-02-20T21:47:16.517 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101768.log 2026-02-20T21:47:16.517 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50447.log 2026-02-20T21:47:16.517 INFO:teuthology.orchestra.run.trial163.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.134350.log.gz 2026-02-20T21:47:16.517 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141522.log 2026-02-20T21:47:16.517 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50447.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72189.log 2026-02-20T21:47:16.517 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50447.log.gz 2026-02-20T21:47:16.522 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141522.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.103897.log.gz 2026-02-20T21:47:16.524 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69392.log 2026-02-20T21:47:16.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.72189.log: 88.4% -- replaced with /var/log/ceph/cluster1-client.mirror.72189.log.gz 2026-02-20T21:47:16.525 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77415.log 2026-02-20T21:47:16.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101768.log: /var/log/ceph/cluster2-client.mirror.69392.log: 88.7% 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.101768.log.gz 2026-02-20T21:47:16.525 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.69392.log.gz 2026-02-20T21:47:16.525 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74413.log 2026-02-20T21:47:16.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77415.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77415.log.gz 2026-02-20T21:47:16.526 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134117.log 2026-02-20T21:47:16.527 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77322.log 2026-02-20T21:47:16.529 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.74413.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74413.log.gz 2026-02-20T21:47:16.530 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134117.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134117.log.gz 2026-02-20T21:47:16.530 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67439.log 2026-02-20T21:47:16.532 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77322.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77322.log.gz 2026-02-20T21:47:16.532 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137149.log 2026-02-20T21:47:16.532 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67439.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67439.log.gz 2026-02-20T21:47:16.535 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120176.log 2026-02-20T21:47:16.535 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137149.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137149.log.gz 2026-02-20T21:47:16.535 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106994.log 2026-02-20T21:47:16.535 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120176.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120176.log.gz 2026-02-20T21:47:16.535 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100726.log 2026-02-20T21:47:16.537 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.106994.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.106994.log.gz 2026-02-20T21:47:16.537 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102078.log 2026-02-20T21:47:16.537 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100726.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.100726.log.gz 2026-02-20T21:47:16.538 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79054.log 2026-02-20T21:47:16.539 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102078.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.102078.log.gz 2026-02-20T21:47:16.539 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120862.log 2026-02-20T21:47:16.540 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79054.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79054.log.gz 2026-02-20T21:47:16.542 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61669.log 2026-02-20T21:47:16.542 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120862.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.120862.log.gz 2026-02-20T21:47:16.542 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110100.log 2026-02-20T21:47:16.542 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61669.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61669.log.gz 2026-02-20T21:47:16.542 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67381.log 2026-02-20T21:47:16.548 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110100.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.110100.log.gz 2026-02-20T21:47:16.548 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.113161.log.gz 2026-02-20T21:47:16.548 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101986.log 2026-02-20T21:47:16.548 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137397.log 2026-02-20T21:47:16.548 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67381.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67381.log.gz 2026-02-20T21:47:16.548 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101986.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.101986.log.gz 2026-02-20T21:47:16.548 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47400.log 2026-02-20T21:47:16.548 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137397.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.137397.log.gzgzip 2026-02-20T21:47:16.548 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133691.log 2026-02-20T21:47:16.548 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.47400.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124659.log 2026-02-20T21:47:16.549 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133691.log: 83.6% -- replaced with /var/log/ceph/cluster1-client.mirror.47400.log.gz 2026-02-20T21:47:16.549 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.133691.log.gz 2026-02-20T21:47:16.549 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96047.log 2026-02-20T21:47:16.549 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120536.log 2026-02-20T21:47:16.549 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124659.log: /var/log/ceph/cluster2-client.mirror.96047.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.96047.log.gz 2026-02-20T21:47:16.549 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74456.log 2026-02-20T21:47:16.549 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120536.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120536.log.gz 2026-02-20T21:47:16.549 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132503.log 2026-02-20T21:47:16.551 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.74456.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74456.log.gz 2026-02-20T21:47:16.551 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110910.log 2026-02-20T21:47:16.552 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132503.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.132503.log.gz 2026-02-20T21:47:16.553 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62971.log 2026-02-20T21:47:16.553 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110910.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110910.log.gz 2026-02-20T21:47:16.554 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133031.log 2026-02-20T21:47:16.554 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62971.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62971.log.gz 2026-02-20T21:47:16.554 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96351.log 2026-02-20T21:47:16.556 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133031.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.133031.log.gz 2026-02-20T21:47:16.556 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119965.log 2026-02-20T21:47:16.566 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96351.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138342.log 2026-02-20T21:47:16.578 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119965.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119965.log.gz 2026-02-20T21:47:16.579 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111064.log 2026-02-20T21:47:16.580 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138342.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.138342.log.gz 2026-02-20T21:47:16.586 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100172.log 2026-02-20T21:47:16.587 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111064.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.111064.log.gz 2026-02-20T21:47:16.599 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118101.log 2026-02-20T21:47:16.600 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100172.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100172.log.gz 2026-02-20T21:47:16.613 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91830.log.gz 2026-02-20T21:47:16.613 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89027.log 2026-02-20T21:47:16.618 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118101.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118101.log.gz 2026-02-20T21:47:16.618 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89599.log 2026-02-20T21:47:16.619 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89027.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89027.log.gz 2026-02-20T21:47:16.619 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86593.log 2026-02-20T21:47:16.619 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.89599.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89599.log.gz 2026-02-20T21:47:16.619 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84029.log 2026-02-20T21:47:16.619 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86593.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.86593.log.gz 2026-02-20T21:47:16.619 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82216.log 2026-02-20T21:47:16.627 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84029.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93608.log 2026-02-20T21:47:16.627 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82216.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82216.log.gz 2026-02-20T21:47:16.639 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.116479.log 2026-02-20T21:47:16.640 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93608.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.93608.log.gz 2026-02-20T21:47:16.646 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68276.log 2026-02-20T21:47:16.656 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.116479.log: 92.5% 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.107016.log.gz 2026-02-20T21:47:16.661 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120819.log 2026-02-20T21:47:16.661 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68276.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68276.log.gz 2026-02-20T21:47:16.661 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131914.log 2026-02-20T21:47:16.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120819.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.120819.log.gz 2026-02-20T21:47:16.662 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83804.log 2026-02-20T21:47:16.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131914.log: -- replaced with /var/log/ceph/cluster1-client.mirror.116479.log.gz 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.131914.log.gz 2026-02-20T21:47:16.662 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:16.662 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47867.log 2026-02-20T21:47:16.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.83804.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.mirror.83804.log.gz 2026-02-20T21:47:16.667 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70140.log 2026-02-20T21:47:16.667 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.47867.log: 63.8% -- replaced with /var/log/ceph/cluster1-client.mirror.47867.log.gz 2026-02-20T21:47:16.667 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114251.log 2026-02-20T21:47:16.667 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70140.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70140.log.gz 2026-02-20T21:47:16.668 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60508.log 2026-02-20T21:47:16.668 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114251.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.114251.log.gz 2026-02-20T21:47:16.668 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88403.log 2026-02-20T21:47:16.668 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.60508.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.60508.log.gz 2026-02-20T21:47:16.668 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139640.log 2026-02-20T21:47:16.672 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88403.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88403.log.gz 2026-02-20T21:47:16.672 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135427.log 2026-02-20T21:47:16.672 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139640.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.139640.log.gz 2026-02-20T21:47:16.672 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57824.log 2026-02-20T21:47:16.672 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135427.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.135427.log.gz 2026-02-20T21:47:16.675 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135710.log 2026-02-20T21:47:16.675 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57824.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57824.log.gz 2026-02-20T21:47:16.676 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65259.log 2026-02-20T21:47:16.676 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135710.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.135710.log.gz 2026-02-20T21:47:16.676 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139749.log 2026-02-20T21:47:16.684 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65259.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120742.log 2026-02-20T21:47:16.697 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139749.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.139749.log.gz 2026-02-20T21:47:16.701 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138252.log 2026-02-20T21:47:16.702 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120742.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.120742.log.gz 2026-02-20T21:47:16.708 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52837.log 2026-02-20T21:47:16.709 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138252.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138252.log.gz 2026-02-20T21:47:16.719 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91612.log.gz 2026-02-20T21:47:16.720 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141841.log 2026-02-20T21:47:16.727 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52837.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52837.log.gz 2026-02-20T21:47:16.727 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132798.log 2026-02-20T21:47:16.727 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141841.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.141841.log.gz 2026-02-20T21:47:16.727 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92858.log 2026-02-20T21:47:16.727 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132798.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.132798.log.gz 2026-02-20T21:47:16.727 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109630.log 2026-02-20T21:47:16.728 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92858.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.92858.log.gz 2026-02-20T21:47:16.728 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70233.log 2026-02-20T21:47:16.728 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109630.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.109630.log.gz 2026-02-20T21:47:16.728 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108310.log 2026-02-20T21:47:16.728 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70233.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70233.log.gz 2026-02-20T21:47:16.728 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130086.log 2026-02-20T21:47:16.728 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.108310.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.108310.log.gz 2026-02-20T21:47:16.728 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66079.log 2026-02-20T21:47:16.733 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130086.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90441.log.gz 2026-02-20T21:47:16.735 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115187.log 2026-02-20T21:47:16.735 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66079.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66079.log.gz 2026-02-20T21:47:16.735 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98453.log 2026-02-20T21:47:16.738 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115187.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115187.log.gz 2026-02-20T21:47:16.738 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110945.log 2026-02-20T21:47:16.738 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98453.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98453.log.gz 2026-02-20T21:47:16.738 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47901.log 2026-02-20T21:47:16.738 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110945.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.110945.log.gz 2026-02-20T21:47:16.744 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134472.log 2026-02-20T21:47:16.745 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.47901.log: 53.8% -- replaced with /var/log/ceph/cluster1-client.mirror.47901.log.gz 2026-02-20T21:47:16.745 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121355.log 2026-02-20T21:47:16.745 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134472.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134472.log.gz 2026-02-20T21:47:16.745 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56272.log 2026-02-20T21:47:16.745 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121355.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121355.log.gz 2026-02-20T21:47:16.745 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142147.log 2026-02-20T21:47:16.745 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56272.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.56272.log.gz 2026-02-20T21:47:16.745 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88141.log 2026-02-20T21:47:16.745 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142147.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.142147.log.gz 2026-02-20T21:47:16.746 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76746.log 2026-02-20T21:47:16.752 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88141.log: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.88141.log.gz 2026-02-20T21:47:16.752 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72957.log 2026-02-20T21:47:16.752 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.76746.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.76746.log.gz 2026-02-20T21:47:16.752 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94337.log 2026-02-20T21:47:16.752 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72957.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72957.log.gz 2026-02-20T21:47:16.752 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133300.log 2026-02-20T21:47:16.753 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94337.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.94337.log.gz 2026-02-20T21:47:16.753 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133391.log 2026-02-20T21:47:16.762 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133300.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.133300.log.gz 2026-02-20T21:47:16.763 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138432.log 2026-02-20T21:47:16.763 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133391.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.133391.log.gz 2026-02-20T21:47:16.763 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138229.log 2026-02-20T21:47:16.763 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138432.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138432.log.gz 2026-02-20T21:47:16.763 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112308.log 2026-02-20T21:47:16.763 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138229.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138229.log.gz 2026-02-20T21:47:16.763 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68927.log 2026-02-20T21:47:16.763 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112308.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112308.log.gz 2026-02-20T21:47:16.763 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103141.log 2026-02-20T21:47:16.764 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68927.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68927.log.gz 2026-02-20T21:47:16.764 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117537.log 2026-02-20T21:47:16.764 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103141.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.103141.log.gz 2026-02-20T21:47:16.764 INFO:teuthology.orchestra.run.trial163.stderr: 95.8% -- replaced with /var/log/ceph/cluster2-client.mirror.90769.log.gz 2026-02-20T21:47:16.764 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84077.log 2026-02-20T21:47:16.764 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87199.log 2026-02-20T21:47:16.764 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117537.log: /var/log/ceph/cluster2-client.mirror.84077.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117537.log.gz 2026-02-20T21:47:16.764 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102982.log 2026-02-20T21:47:16.764 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87199.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87199.log.gz 2026-02-20T21:47:16.764 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121541.log 2026-02-20T21:47:16.764 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102982.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.102982.log.gz 2026-02-20T21:47:16.764 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102486.log 2026-02-20T21:47:16.768 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121541.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121541.log.gz 2026-02-20T21:47:16.768 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63494.log 2026-02-20T21:47:16.768 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102486.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.102486.log.gz 2026-02-20T21:47:16.768 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48260.log 2026-02-20T21:47:16.768 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63494.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63494.log.gz 2026-02-20T21:47:16.768 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90852.log 2026-02-20T21:47:16.768 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.48260.log: 80.1% -- replaced with /var/log/ceph/cluster2-client.admin.48260.log.gz 2026-02-20T21:47:16.768 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78624.log 2026-02-20T21:47:16.776 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90852.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.138743.log 2026-02-20T21:47:16.778 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78624.log: 88.7% 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91984.log.gz 2026-02-20T21:47:16.778 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.78624.log.gz 2026-02-20T21:47:16.780 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81414.log 2026-02-20T21:47:16.780 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.138743.log: 63.2% -- replaced with /var/log/ceph/cluster2-client.mirror.138743.log.gz 2026-02-20T21:47:16.780 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.143214.log 2026-02-20T21:47:16.780 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81414.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81414.log.gz 2026-02-20T21:47:16.782 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55461.log 2026-02-20T21:47:16.782 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.143214.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.143214.log.gz 2026-02-20T21:47:16.782 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62506.log 2026-02-20T21:47:16.782 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55461.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54138.log.gz 2026-02-20T21:47:16.782 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.55461.log.gz 2026-02-20T21:47:16.783 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108818.log 2026-02-20T21:47:16.783 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62506.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74956.log 2026-02-20T21:47:16.783 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.62506.log.gz 2026-02-20T21:47:16.784 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108818.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.108818.log.gz 2026-02-20T21:47:16.791 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.109144.log 2026-02-20T21:47:16.791 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74956.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91538.log 2026-02-20T21:47:16.791 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74956.log.gz 2026-02-20T21:47:16.791 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.3.109144.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111022.log 2026-02-20T21:47:16.796 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91538.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84077.log.gz 2026-02-20T21:47:16.797 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.3.109144.log.gz 2026-02-20T21:47:16.797 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122466.log 2026-02-20T21:47:16.797 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111022.log: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.131707.log 2026-02-20T21:47:16.797 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.111022.log.gz 2026-02-20T21:47:16.797 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122466.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85750.log 2026-02-20T21:47:16.800 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.131707.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.122466.log.gz 2026-02-20T21:47:16.800 INFO:teuthology.orchestra.run.trial163.stderr: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.131707.log.gz 2026-02-20T21:47:16.800 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109401.log 2026-02-20T21:47:16.800 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49910.log 2026-02-20T21:47:16.800 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85750.log: /var/log/ceph/cluster2-client.mirror.109401.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.85750.log.gz 2026-02-20T21:47:16.800 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.109401.log.gz 2026-02-20T21:47:16.800 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92260.log 2026-02-20T21:47:16.800 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127253.log 2026-02-20T21:47:16.800 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49910.log: /var/log/ceph/cluster2-client.mirror.92260.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.92260.log.gz 2026-02-20T21:47:16.800 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.49910.log.gz 2026-02-20T21:47:16.801 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141203.log 2026-02-20T21:47:16.802 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.143312.log 2026-02-20T21:47:16.802 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127253.log: /var/log/ceph/cluster2-client.mirror.141203.log: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127253.log.gz 2026-02-20T21:47:16.802 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108586.log 2026-02-20T21:47:16.803 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.143312.log: 0.0% 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.141203.log.gz 2026-02-20T21:47:16.803 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.143312.log.gz 2026-02-20T21:47:16.810 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137217.log 2026-02-20T21:47:16.810 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108586.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65194.log 2026-02-20T21:47:16.810 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108586.log.gz 2026-02-20T21:47:16.810 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137217.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89647.log 2026-02-20T21:47:16.811 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65194.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137217.log.gz 2026-02-20T21:47:16.811 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136744.log 2026-02-20T21:47:16.812 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89647.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65194.log.gz 2026-02-20T21:47:16.813 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52894.log 2026-02-20T21:47:16.815 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136744.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.136744.log.gz 2026-02-20T21:47:16.815 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113618.log 2026-02-20T21:47:16.815 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52894.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77845.log 2026-02-20T21:47:16.818 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.52894.log.gz 2026-02-20T21:47:16.820 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113618.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113618.log.gz 2026-02-20T21:47:16.820 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128191.log 2026-02-20T21:47:16.820 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77845.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77845.log.gz 2026-02-20T21:47:16.823 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107116.log 2026-02-20T21:47:16.823 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128191.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128191.log.gz 2026-02-20T21:47:16.823 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90007.log 2026-02-20T21:47:16.823 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107116.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107116.log.gz 2026-02-20T21:47:16.823 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132667.log 2026-02-20T21:47:16.832 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90007.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64424.log 2026-02-20T21:47:16.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132667.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59667.log 2026-02-20T21:47:16.845 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132667.log.gz 2026-02-20T21:47:16.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64424.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64424.log.gz 2026-02-20T21:47:16.846 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53049.log.gz 2026-02-20T21:47:16.848 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116566.log 2026-02-20T21:47:16.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59667.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59667.log.gz 2026-02-20T21:47:16.849 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66044.log 2026-02-20T21:47:16.853 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116566.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.116566.log.gz 2026-02-20T21:47:16.853 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100430.log 2026-02-20T21:47:16.853 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66044.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66044.log.gz 2026-02-20T21:47:16.853 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62413.log 2026-02-20T21:47:16.853 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100430.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100430.log.gz 2026-02-20T21:47:16.854 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134634.log 2026-02-20T21:47:16.854 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62413.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62413.log.gz 2026-02-20T21:47:16.854 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114166.log 2026-02-20T21:47:16.861 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134634.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.134634.log.gz 2026-02-20T21:47:16.862 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123516.log 2026-02-20T21:47:16.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114166.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114166.log.gz 2026-02-20T21:47:16.862 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134756.log 2026-02-20T21:47:16.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123516.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123516.log.gz 2026-02-20T21:47:16.862 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48379.log 2026-02-20T21:47:16.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134756.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.134756.log.gz 2026-02-20T21:47:16.862 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67102.log 2026-02-20T21:47:16.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48379.log: 64.8% -- replaced with /var/log/ceph/cluster2-client.mirror.48379.log.gz 2026-02-20T21:47:16.862 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54685.log 2026-02-20T21:47:16.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67102.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67102.log.gz 2026-02-20T21:47:16.862 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90517.log 2026-02-20T21:47:16.863 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54685.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.54685.log.gz 2026-02-20T21:47:16.863 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75107.log 2026-02-20T21:47:16.865 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90517.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.100860.log.gz 2026-02-20T21:47:16.868 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127067.log 2026-02-20T21:47:16.868 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75107.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75107.log.gz 2026-02-20T21:47:16.868 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73422.log 2026-02-20T21:47:16.868 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127067.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127067.log.gz 2026-02-20T21:47:16.868 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92395.log 2026-02-20T21:47:16.868 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73422.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73422.log.gz 2026-02-20T21:47:16.877 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112744.log 2026-02-20T21:47:16.881 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.92395.log: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92395.log.gz 2026-02-20T21:47:16.881 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118465.log 2026-02-20T21:47:16.882 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112744.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112744.log.gz 2026-02-20T21:47:16.882 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.114665.log 2026-02-20T21:47:16.883 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118465.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.118465.log.gz 2026-02-20T21:47:16.883 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73592.log 2026-02-20T21:47:16.890 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.3.114665.log: 94.2% -- replaced with /var/log/ceph/cluster2-client.mirror.90632.log.gz 2026-02-20T21:47:16.891 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.65321.log 2026-02-20T21:47:16.891 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73592.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73592.log.gz 2026-02-20T21:47:16.891 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84534.log 2026-02-20T21:47:16.898 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.65321.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.0.65321.log.gz 2026-02-20T21:47:16.899 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115543.log 2026-02-20T21:47:16.899 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84534.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84534.log.gz 2026-02-20T21:47:16.904 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81693.log 2026-02-20T21:47:16.904 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115543.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.115543.log.gz 2026-02-20T21:47:16.904 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47020.log 2026-02-20T21:47:16.904 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81693.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81693.log.gz 2026-02-20T21:47:16.904 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73624.log 2026-02-20T21:47:16.904 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.47020.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47020.log.gz 2026-02-20T21:47:16.904 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57498.log 2026-02-20T21:47:16.905 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.73624.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73624.log.gz 2026-02-20T21:47:16.905 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120366.log 2026-02-20T21:47:16.905 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57498.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57498.log.gz 2026-02-20T21:47:16.905 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74224.log 2026-02-20T21:47:16.905 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120366.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120366.log.gz 2026-02-20T21:47:16.905 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139516.log 2026-02-20T21:47:16.905 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74224.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74224.log.gz 2026-02-20T21:47:16.908 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141151.log 2026-02-20T21:47:16.908 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139516.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.139516.log.gz 2026-02-20T21:47:16.908 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86896.log 2026-02-20T21:47:16.909 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.141151.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.141151.log.gz 2026-02-20T21:47:16.909 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73480.log 2026-02-20T21:47:16.909 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86896.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.86896.log.gz 2026-02-20T21:47:16.909 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48396.log 2026-02-20T21:47:16.909 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73480.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73480.log.gz 2026-02-20T21:47:16.909 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78345.log 2026-02-20T21:47:16.912 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48396.log: 68.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48396.log.gz 2026-02-20T21:47:16.912 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103460.log 2026-02-20T21:47:16.912 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78345.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78345.log.gz 2026-02-20T21:47:16.912 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137532.log 2026-02-20T21:47:16.912 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103460.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.103460.log.gz 2026-02-20T21:47:16.912 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108377.log 2026-02-20T21:47:16.913 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137532.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137532.log.gz 2026-02-20T21:47:16.913 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140360.log 2026-02-20T21:47:16.921 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108377.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67753.log 2026-02-20T21:47:16.922 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.140360.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.140360.log.gz 2026-02-20T21:47:16.925 INFO:teuthology.orchestra.run.trial163.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.3.114665.log.gz 2026-02-20T21:47:16.925 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47273.log 2026-02-20T21:47:16.926 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67753.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67753.log.gz 2026-02-20T21:47:16.926 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142535.log 2026-02-20T21:47:16.927 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47273.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.mirror.47273.log.gz 2026-02-20T21:47:16.927 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.142301.log 2026-02-20T21:47:16.927 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142535.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.142535.log.gz 2026-02-20T21:47:16.928 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76436.log 2026-02-20T21:47:16.928 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.142301.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.142301.log.gz 2026-02-20T21:47:16.929 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132319.log 2026-02-20T21:47:16.929 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.76436.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.mirror.76436.log.gz 2026-02-20T21:47:16.930 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140766.log 2026-02-20T21:47:16.930 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132319.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132319.log.gz 2026-02-20T21:47:16.930 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101936.log 2026-02-20T21:47:16.931 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140766.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.140766.log.gz 2026-02-20T21:47:16.931 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96284.log 2026-02-20T21:47:16.931 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101936.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101936.log.gz 2026-02-20T21:47:16.932 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88790.log 2026-02-20T21:47:16.932 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.96284.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.96284.log.gz 2026-02-20T21:47:16.933 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85987.log 2026-02-20T21:47:16.936 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88790.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88790.log.gz 2026-02-20T21:47:16.937 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109436.log 2026-02-20T21:47:16.937 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85987.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85987.log.gz 2026-02-20T21:47:16.938 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-20T21:47:16.938 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109436.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.109436.log.gz 2026-02-20T21:47:16.938 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115005.log 2026-02-20T21:47:16.939 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73838.log 2026-02-20T21:47:16.951 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115005.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114366.log 2026-02-20T21:47:16.952 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73838.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115005.log.gz 2026-02-20T21:47:16.964 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136834.log 2026-02-20T21:47:16.965 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114366.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.114366.log.gz 2026-02-20T21:47:16.978 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48413.log 2026-02-20T21:47:16.978 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136834.log: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.136834.log.gz 2026-02-20T21:47:16.979 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59907.log 2026-02-20T21:47:16.980 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48413.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60567.log.gz 2026-02-20T21:47:16.980 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134827.log 2026-02-20T21:47:16.980 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59907.log: 55.6% -- replaced with /var/log/ceph/cluster2-client.mirror.48413.log.gz 2026-02-20T21:47:16.980 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.59907.log.gz 2026-02-20T21:47:16.990 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109886.log 2026-02-20T21:47:16.991 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134827.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.134827.log.gz 2026-02-20T21:47:17.007 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137172.log 2026-02-20T21:47:17.009 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109886.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.109886.log.gz 2026-02-20T21:47:17.026 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60646.log 2026-02-20T21:47:17.026 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137172.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137172.log.gz 2026-02-20T21:47:17.034 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55655.log 2026-02-20T21:47:17.044 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60646.log: 88.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63122.log 2026-02-20T21:47:17.046 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55655.log: 88.5% 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.124659.log.gz 2026-02-20T21:47:17.047 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.60646.log.gz 2026-02-20T21:47:17.047 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136542.log 2026-02-20T21:47:17.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63122.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63122.log.gz 2026-02-20T21:47:17.049 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134776.log 2026-02-20T21:47:17.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136542.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.136542.log.gz 2026-02-20T21:47:17.049 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55267.log 2026-02-20T21:47:17.052 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134776.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.134776.log.gz 2026-02-20T21:47:17.053 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100793.log 2026-02-20T21:47:17.053 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55267.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55267.log.gz 2026-02-20T21:47:17.053 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.138910.log 2026-02-20T21:47:17.053 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.100793.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.100793.log.gz 2026-02-20T21:47:17.053 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.55655.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107294.log 2026-02-20T21:47:17.053 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:17.055 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.138910.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106375.log 2026-02-20T21:47:17.063 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107294.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91233.log.gz 2026-02-20T21:47:17.064 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101019.log 2026-02-20T21:47:17.067 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.106375.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.106375.log.gz 2026-02-20T21:47:17.067 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107294.log.gz 2026-02-20T21:47:17.067 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75142.log 2026-02-20T21:47:17.068 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101019.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101019.log.gz 2026-02-20T21:47:17.068 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141391.log 2026-02-20T21:47:17.068 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75142.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75142.log.gz 2026-02-20T21:47:17.068 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74592.log 2026-02-20T21:47:17.068 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141391.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.141391.log.gz 2026-02-20T21:47:17.068 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67811.log 2026-02-20T21:47:17.082 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74592.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56140.log 2026-02-20T21:47:17.083 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67811.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67811.log.gz 2026-02-20T21:47:17.096 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118228.log 2026-02-20T21:47:17.097 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56140.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56140.log.gz 2026-02-20T21:47:17.111 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135080.log 2026-02-20T21:47:17.111 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118228.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.118228.log.gz 2026-02-20T21:47:17.120 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107151.log 2026-02-20T21:47:17.121 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135080.log: 90.6% 95.2% -- replaced with /var/log/ceph/cluster2-client.mirror.138910.log.gz 2026-02-20T21:47:17.123 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126097.log 2026-02-20T21:47:17.124 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107151.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.107151.log.gz 2026-02-20T21:47:17.124 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94986.log 2026-02-20T21:47:17.134 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126097.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49146.log 2026-02-20T21:47:17.135 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.94986.log: 2.6% -- replaced with /var/log/ceph/cluster1-client.mirror.94986.log.gz 2026-02-20T21:47:17.135 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.135080.log.gz 2026-02-20T21:47:17.142 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92910.log 2026-02-20T21:47:17.143 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49146.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.96351.log.gz 2026-02-20T21:47:17.159 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93631.log.gz 2026-02-20T21:47:17.159 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81472.log 2026-02-20T21:47:17.159 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92910.log: 82.8% -- replaced with /var/log/ceph/cluster2-client.mirror.92910.log.gz 2026-02-20T21:47:17.159 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107751.log 2026-02-20T21:47:17.159 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81472.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81472.log.gz 2026-02-20T21:47:17.159 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76524.log 2026-02-20T21:47:17.159 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107751.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107751.log.gz 2026-02-20T21:47:17.159 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139079.log 2026-02-20T21:47:17.173 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.76524.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.76524.log.gz 2026-02-20T21:47:17.173 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112690.log 2026-02-20T21:47:17.173 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139079.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.139079.log.gz 2026-02-20T21:47:17.173 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85583.log 2026-02-20T21:47:17.173 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112690.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112690.log.gz 2026-02-20T21:47:17.173 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136969.log 2026-02-20T21:47:17.173 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85583.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.85583.log.gz 2026-02-20T21:47:17.174 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123197.log 2026-02-20T21:47:17.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136969.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.136969.log.gz 2026-02-20T21:47:17.174 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118824.log 2026-02-20T21:47:17.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.123197.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.123197.log.gz 2026-02-20T21:47:17.174 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120141.log 2026-02-20T21:47:17.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118824.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.118824.log.gz 2026-02-20T21:47:17.174 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125776.log 2026-02-20T21:47:17.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120141.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.120141.log.gz 2026-02-20T21:47:17.174 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80763.log 2026-02-20T21:47:17.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125776.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125776.log.gz 2026-02-20T21:47:17.174 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70640.log 2026-02-20T21:47:17.174 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80763.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80763.log.gz 2026-02-20T21:47:17.175 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140385.log 2026-02-20T21:47:17.175 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70640.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.70640.log.gz 2026-02-20T21:47:17.175 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93183.log 2026-02-20T21:47:17.176 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140385.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.140385.log.gz 2026-02-20T21:47:17.177 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84942.log 2026-02-20T21:47:17.183 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93183.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122374.log 2026-02-20T21:47:17.196 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84942.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.84942.log.gz 2026-02-20T21:47:17.196 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91538.log.gz 2026-02-20T21:47:17.201 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119590.log 2026-02-20T21:47:17.201 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122374.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122374.log.gz 2026-02-20T21:47:17.201 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112093.log 2026-02-20T21:47:17.201 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119590.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119590.log.gz 2026-02-20T21:47:17.201 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.47273.log 2026-02-20T21:47:17.201 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112093.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112093.log.gz 2026-02-20T21:47:17.201 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74770.log 2026-02-20T21:47:17.201 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.3.47273.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76481.log 2026-02-20T21:47:17.209 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81937.log 2026-02-20T21:47:17.210 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74770.log: /var/log/ceph/cluster1-client.mirror.76481.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74770.log.gz 2026-02-20T21:47:17.213 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.76481.log.gz 2026-02-20T21:47:17.239 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.92564.log.gz 2026-02-20T21:47:17.239 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46677.log 2026-02-20T21:47:17.239 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81937.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81937.log.gz 2026-02-20T21:47:17.240 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51217.log 2026-02-20T21:47:17.240 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.46677.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.74501.log.gz 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46677.log.gz 2026-02-20T21:47:17.240 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:17.240 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62285.log 2026-02-20T21:47:17.240 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51217.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65707.log 2026-02-20T21:47:17.240 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62285.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51217.log.gz 2026-02-20T21:47:17.240 INFO:teuthology.orchestra.run.trial163.stderr: 88.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89375.log 2026-02-20T21:47:17.240 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.62285.log.gz 2026-02-20T21:47:17.240 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65707.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65707.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98713.log 2026-02-20T21:47:17.240 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:17.240 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125590.log 2026-02-20T21:47:17.240 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89375.log: /var/log/ceph/cluster2-client.mirror.98713.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98713.log.gz 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.89375.log.gz 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121278.log 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125590.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63587.log 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125590.log.gz 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121278.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71849.log 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63587.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121278.log.gz 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63587.log.gz 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.100838.log 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.71849.log: /var/log/ceph/cluster2-client.mirror.80728.log 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71849.log.gz 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.100838.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.100838.log.gz 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102947.log 2026-02-20T21:47:17.241 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80728.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140277.log 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80728.log.gz 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102947.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102947.log.gz 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.132432.log 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140277.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139916.log 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.132432.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.132432.log.gz 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99525.log 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139916.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.139916.log.gz 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122657.log 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99525.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99525.log.gz 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56431.log 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122657.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.122657.log.gz 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83506.log 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56431.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56431.log.gz 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100592.log 2026-02-20T21:47:17.242 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83506.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.83506.log.gz 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47816.log 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100592.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100592.log.gz 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80670.log 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.47816.log: 5.8% -- replaced with /var/log/ceph/cluster1-client.mirror.47816.log.gz 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106650.log 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80670.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.80670.log.gz 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137577.log 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106650.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106650.log.gz 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49232.log 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137577.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137577.log.gz 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122126.log 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.49232.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134734.log 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.49232.log.gz 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122126.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.122126.log.gz 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141901.log 2026-02-20T21:47:17.243 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134734.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134734.log.gz 2026-02-20T21:47:17.244 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63401.log 2026-02-20T21:47:17.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141901.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.141901.log.gz 2026-02-20T21:47:17.244 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52659.log 2026-02-20T21:47:17.244 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63401.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63401.log.gz 2026-02-20T21:47:17.247 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.138957.log 2026-02-20T21:47:17.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52659.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52659.log.gz 2026-02-20T21:47:17.247 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71535.log 2026-02-20T21:47:17.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.2.138957.log: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.2.138957.log.gz 2026-02-20T21:47:17.247 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73387.log 2026-02-20T21:47:17.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71535.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71535.log.gz 2026-02-20T21:47:17.247 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71198.log 2026-02-20T21:47:17.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73387.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73387.log.gz 2026-02-20T21:47:17.249 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105379.log 2026-02-20T21:47:17.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71198.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71198.log.gz 2026-02-20T21:47:17.249 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116402.log 2026-02-20T21:47:17.249 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105379.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105379.log.gz 2026-02-20T21:47:17.249 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52802.log 2026-02-20T21:47:17.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116402.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.116402.log.gz 2026-02-20T21:47:17.251 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49490.log 2026-02-20T21:47:17.251 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52802.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140229.log 2026-02-20T21:47:17.251 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.52802.log.gz 2026-02-20T21:47:17.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49490.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49490.log.gz 2026-02-20T21:47:17.252 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112170.log 2026-02-20T21:47:17.252 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140229.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.140229.log.gz 2026-02-20T21:47:17.253 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87858.log 2026-02-20T21:47:17.254 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.112170.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.126097.log.gz 2026-02-20T21:47:17.255 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125186.log 2026-02-20T21:47:17.255 INFO:teuthology.orchestra.run.trial163.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.112170.log.gz/var/log/ceph/cluster2-client.mirror.87858.log: 2026-02-20T21:47:17.255 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87858.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60062.log 2026-02-20T21:47:17.255 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:17.256 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125186.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125186.log.gz 2026-02-20T21:47:17.256 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131369.log 2026-02-20T21:47:17.256 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140664.log 2026-02-20T21:47:17.256 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.60062.log: /var/log/ceph/cluster2-client.mirror.131369.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131369.log.gz 2026-02-20T21:47:17.259 INFO:teuthology.orchestra.run.trial163.stderr: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.60062.log.gz 2026-02-20T21:47:17.259 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138567.log 2026-02-20T21:47:17.262 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113387.log 2026-02-20T21:47:17.262 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.140664.log: /var/log/ceph/cluster1-client.mirror.138567.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.140664.log.gz 2026-02-20T21:47:17.262 INFO:teuthology.orchestra.run.trial163.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.138567.log.gz 2026-02-20T21:47:17.262 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105954.log 2026-02-20T21:47:17.262 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.113387.log: -- /var/log/ceph/cluster2-client.mirror.93920.log 2026-02-20T21:47:17.262 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113387.log.gz 2026-02-20T21:47:17.262 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105954.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106545.log 2026-02-20T21:47:17.263 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93920.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105954.log.gz 2026-02-20T21:47:17.263 INFO:teuthology.orchestra.run.trial163.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.93920.log.gz 2026-02-20T21:47:17.263 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52570.log 2026-02-20T21:47:17.263 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106545.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.106545.log.gz 2026-02-20T21:47:17.264 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134208.log 2026-02-20T21:47:17.264 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80949.log 2026-02-20T21:47:17.264 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52570.log: /var/log/ceph/cluster2-client.mirror.134208.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131960.log 2026-02-20T21:47:17.264 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52570.log.gz 2026-02-20T21:47:17.265 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80949.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80949.log.gz 2026-02-20T21:47:17.265 INFO:teuthology.orchestra.run.trial163.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.134208.log.gz 2026-02-20T21:47:17.265 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126176.log 2026-02-20T21:47:17.265 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131960.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109829.log 2026-02-20T21:47:17.265 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131960.log.gz 2026-02-20T21:47:17.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97679.log 2026-02-20T21:47:17.266 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126176.log.gz 2026-02-20T21:47:17.266 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109829.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109829.log.gz 2026-02-20T21:47:17.266 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125998.log 2026-02-20T21:47:17.266 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79705.log 2026-02-20T21:47:17.267 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97679.log: /var/log/ceph/cluster2-client.mirror.125998.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.125998.log.gz 2026-02-20T21:47:17.267 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101803.log 2026-02-20T21:47:17.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79705.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79705.log.gz 2026-02-20T21:47:17.269 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43204.log 2026-02-20T21:47:17.269 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101803.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101803.log.gz 2026-02-20T21:47:17.270 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56369.log 2026-02-20T21:47:17.270 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.43204.log: 26.9% -- replaced with /var/log/ceph/cluster2-client.admin.43204.log.gz 2026-02-20T21:47:17.270 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137734.log 2026-02-20T21:47:17.272 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56369.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.56369.log.gz 2026-02-20T21:47:17.272 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.115162.log 2026-02-20T21:47:17.272 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137734.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137734.log.gz 2026-02-20T21:47:17.272 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108335.log 2026-02-20T21:47:17.273 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.115162.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.115162.log.gz 2026-02-20T21:47:17.273 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87385.log 2026-02-20T21:47:17.274 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108335.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.108335.log.gz 2026-02-20T21:47:17.275 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50015.log 2026-02-20T21:47:17.275 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87385.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87385.log.gz 2026-02-20T21:47:17.276 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113730.log 2026-02-20T21:47:17.277 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50015.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116008.log 2026-02-20T21:47:17.277 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50015.log.gz 2026-02-20T21:47:17.278 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.113730.log: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.113730.log.gz 2026-02-20T21:47:17.278 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.60152.log 2026-02-20T21:47:17.279 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116008.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.116008.log.gz 2026-02-20T21:47:17.279 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112655.log 2026-02-20T21:47:17.283 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.1.60152.log: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.1.60152.log.gz 2026-02-20T21:47:17.284 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140895.log 2026-02-20T21:47:17.284 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112655.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136378.log 2026-02-20T21:47:17.285 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.112655.log.gz 2026-02-20T21:47:17.285 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53892.log 2026-02-20T21:47:17.286 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136378.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.136378.log.gz 2026-02-20T21:47:17.286 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63436.log 2026-02-20T21:47:17.287 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53892.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53892.log.gz 2026-02-20T21:47:17.287 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70791.log 2026-02-20T21:47:17.288 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63436.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63436.log.gz 2026-02-20T21:47:17.288 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112058.log 2026-02-20T21:47:17.288 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70791.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70791.log.gz 2026-02-20T21:47:17.288 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77229.log 2026-02-20T21:47:17.289 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112058.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112058.log.gz 2026-02-20T21:47:17.289 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134543.log 2026-02-20T21:47:17.290 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77229.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77229.log.gz 2026-02-20T21:47:17.290 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71968.log 2026-02-20T21:47:17.291 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134543.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.134543.log.gz 2026-02-20T21:47:17.291 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112900.log 2026-02-20T21:47:17.292 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71968.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71968.log.gz 2026-02-20T21:47:17.292 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81600.log 2026-02-20T21:47:17.293 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.112900.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.112900.log.gz 2026-02-20T21:47:17.293 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141608.log 2026-02-20T21:47:17.294 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81600.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81600.log.gz 2026-02-20T21:47:17.294 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.143459.log 2026-02-20T21:47:17.295 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.141608.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.141608.log.gz 2026-02-20T21:47:17.295 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132918.log 2026-02-20T21:47:17.295 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.143459.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.143459.log.gz 2026-02-20T21:47:17.296 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60832.log 2026-02-20T21:47:17.297 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132918.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132918.log.gz 2026-02-20T21:47:17.297 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123084.log 2026-02-20T21:47:17.298 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60832.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60832.log.gz 2026-02-20T21:47:17.298 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71570.log 2026-02-20T21:47:17.298 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123084.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.123084.log.gz 2026-02-20T21:47:17.299 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.143067.log 2026-02-20T21:47:17.299 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71570.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71570.log.gz 2026-02-20T21:47:17.300 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97612.log 2026-02-20T21:47:17.300 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.143067.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.143067.log.gz 2026-02-20T21:47:17.301 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54817.log 2026-02-20T21:47:17.301 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.97612.log: /var/log/ceph/cluster1-client.mirror.140895.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.97612.log.gz 2026-02-20T21:47:17.302 INFO:teuthology.orchestra.run.trial163.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.140895.log.gz 2026-02-20T21:47:17.302 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62227.log 2026-02-20T21:47:17.302 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54817.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.54817.log.gz 2026-02-20T21:47:17.302 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.138761.log 2026-02-20T21:47:17.303 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62227.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62227.log.gz 2026-02-20T21:47:17.304 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133882.log 2026-02-20T21:47:17.304 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.138761.log: 83.3% -- replaced with /var/log/ceph/cluster2-client.mirror.138761.log.gz 2026-02-20T21:47:17.304 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86290.log 2026-02-20T21:47:17.305 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133882.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.133882.log.gz 2026-02-20T21:47:17.306 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127133.log 2026-02-20T21:47:17.306 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86290.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86290.log.gz 2026-02-20T21:47:17.307 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62564.log 2026-02-20T21:47:17.307 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.108377.log.gz 2026-02-20T21:47:17.308 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127133.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98499.log 2026-02-20T21:47:17.308 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127133.log.gz 2026-02-20T21:47:17.308 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62564.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92831.log 2026-02-20T21:47:17.308 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62564.log.gz 2026-02-20T21:47:17.308 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98499.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98499.log.gz 2026-02-20T21:47:17.308 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85078.log 2026-02-20T21:47:17.309 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104298.log 2026-02-20T21:47:17.309 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92831.log: /var/log/ceph/cluster2-client.mirror.85078.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.85078.log.gz 2026-02-20T21:47:17.317 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102351.log 2026-02-20T21:47:17.318 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104298.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104298.log.gz 2026-02-20T21:47:17.318 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98301.log 2026-02-20T21:47:17.319 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.102351.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.102351.log.gz 2026-02-20T21:47:17.319 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130381.log 2026-02-20T21:47:17.320 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98301.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98301.log.gz 2026-02-20T21:47:17.320 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116218.log 2026-02-20T21:47:17.333 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130381.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87527.log 2026-02-20T21:47:17.334 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116218.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.116218.log.gz 2026-02-20T21:47:17.342 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.84029.log.gz 2026-02-20T21:47:17.343 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138499.log 2026-02-20T21:47:17.343 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87527.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.130086.log.gz 2026-02-20T21:47:17.344 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83892.log 2026-02-20T21:47:17.344 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138499.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138499.log.gz 2026-02-20T21:47:17.344 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138726.log 2026-02-20T21:47:17.346 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.83892.log: 96.4% -- replaced with /var/log/ceph/cluster2-client.mirror.87527.log.gz 2026-02-20T21:47:17.348 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72214.log 2026-02-20T21:47:17.348 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138726.log: 61.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138726.log.gz 2026-02-20T21:47:17.348 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83892.log.gz 2026-02-20T21:47:17.348 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69710.log 2026-02-20T21:47:17.348 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72214.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71291.log 2026-02-20T21:47:17.349 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69710.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69710.log.gz 2026-02-20T21:47:17.349 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.114433.log 2026-02-20T21:47:17.350 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71291.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71291.log.gz 2026-02-20T21:47:17.350 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129174.log 2026-02-20T21:47:17.351 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.114433.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.114433.log.gz 2026-02-20T21:47:17.351 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118344.log 2026-02-20T21:47:17.351 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72214.log.gz 2026-02-20T21:47:17.354 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129174.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81879.log 2026-02-20T21:47:17.354 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.129174.log.gz 2026-02-20T21:47:17.354 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118344.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118344.log.gz 2026-02-20T21:47:17.354 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141799.log 2026-02-20T21:47:17.354 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133620.log 2026-02-20T21:47:17.354 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81879.log: /var/log/ceph/cluster1-client.mirror.141799.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.141799.log.gz 2026-02-20T21:47:17.354 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81879.log.gz 2026-02-20T21:47:17.354 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47173.log 2026-02-20T21:47:17.354 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133620.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115357.log 2026-02-20T21:47:17.356 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133620.log.gz 2026-02-20T21:47:17.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47173.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51641.log 2026-02-20T21:47:17.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115357.log: 87.6% 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.115357.log.gz 2026-02-20T21:47:17.356 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.47173.log.gz 2026-02-20T21:47:17.356 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82251.log 2026-02-20T21:47:17.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51641.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122280.log 2026-02-20T21:47:17.356 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51641.log.gz/var/log/ceph/cluster2-client.mirror.82251.log: 2026-02-20T21:47:17.359 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82251.log.gz 2026-02-20T21:47:17.359 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69861.log 2026-02-20T21:47:17.359 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122280.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106212.log 2026-02-20T21:47:17.359 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122280.log.gz 2026-02-20T21:47:17.359 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69861.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50050.log 2026-02-20T21:47:17.359 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106212.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69861.log.gz 2026-02-20T21:47:17.359 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106212.log.gz 2026-02-20T21:47:17.359 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96837.log 2026-02-20T21:47:17.359 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.50050.log: -- /var/log/ceph/cluster2-client.mirror.0.47304.log 2026-02-20T21:47:17.361 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96837.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50050.log.gz 2026-02-20T21:47:17.361 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47304.log 2026-02-20T21:47:17.361 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96837.log.gz 2026-02-20T21:47:17.361 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.0.47304.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86861.log 2026-02-20T21:47:17.361 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54293.log 2026-02-20T21:47:17.361 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86861.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86861.log.gz 2026-02-20T21:47:17.362 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46386.log 2026-02-20T21:47:17.362 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54293.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.54293.log.gz 2026-02-20T21:47:17.363 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89543.log 2026-02-20T21:47:17.363 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.46386.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46386.log.gz 2026-02-20T21:47:17.363 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53483.log 2026-02-20T21:47:17.364 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89543.log: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.0.47304.log.gz 2026-02-20T21:47:17.365 INFO:teuthology.orchestra.run.trial163.stderr: 91.1%/var/log/ceph/cluster1-client.mirror.47304.log: -- replaced with /var/log/ceph/cluster2-client.mirror.89543.log.gz 2026-02-20T21:47:17.365 INFO:teuthology.orchestra.run.trial163.stderr: 83.6% -- replaced with /var/log/ceph/cluster1-client.mirror.47304.log.gz 2026-02-20T21:47:17.366 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123130.log 2026-02-20T21:47:17.366 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53483.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79240.log 2026-02-20T21:47:17.367 INFO:teuthology.orchestra.run.trial163.stderr: 87.9%/var/log/ceph/cluster2-client.mirror.123130.log: -- replaced with /var/log/ceph/cluster2-client.mirror.53483.log.gz 2026-02-20T21:47:17.367 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123130.log.gz 2026-02-20T21:47:17.367 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106739.log 2026-02-20T21:47:17.368 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79240.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60465.log 2026-02-20T21:47:17.368 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.79240.log.gz 2026-02-20T21:47:17.368 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106739.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90037.log 2026-02-20T21:47:17.368 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.60465.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.106739.log.gz 2026-02-20T21:47:17.369 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119825.log 2026-02-20T21:47:17.369 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.admin.60465.log.gz 2026-02-20T21:47:17.369 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90037.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61169.log 2026-02-20T21:47:17.369 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119825.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90037.log.gz 2026-02-20T21:47:17.369 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119825.log.gz 2026-02-20T21:47:17.370 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52446.log 2026-02-20T21:47:17.370 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67625.log 2026-02-20T21:47:17.371 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61169.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.61169.log.gz 2026-02-20T21:47:17.371 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52446.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68090.log 2026-02-20T21:47:17.371 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67625.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52446.log.gz 2026-02-20T21:47:17.371 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67625.log.gz 2026-02-20T21:47:17.372 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52713.log 2026-02-20T21:47:17.372 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68090.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74436.log 2026-02-20T21:47:17.372 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.68090.log.gz 2026-02-20T21:47:17.373 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52713.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50797.log 2026-02-20T21:47:17.373 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74436.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.52713.log.gz 2026-02-20T21:47:17.374 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140148.log 2026-02-20T21:47:17.375 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50797.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50797.log.gz 2026-02-20T21:47:17.375 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79182.log 2026-02-20T21:47:17.375 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140148.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.140148.log.gz 2026-02-20T21:47:17.376 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50867.log 2026-02-20T21:47:17.376 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79182.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.74436.log.gz 2026-02-20T21:47:17.377 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79182.log.gz 2026-02-20T21:47:17.377 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132989.log 2026-02-20T21:47:17.377 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50867.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49263.log 2026-02-20T21:47:17.377 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50867.log.gz 2026-02-20T21:47:17.378 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132989.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82030.log 2026-02-20T21:47:17.379 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.49263.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132989.log.gzgzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118497.log 2026-02-20T21:47:17.379 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:17.379 INFO:teuthology.orchestra.run.trial163.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.1.47173.log.gz 2026-02-20T21:47:17.379 INFO:teuthology.orchestra.run.trial163.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.49263.log.gz 2026-02-20T21:47:17.379 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117448.log 2026-02-20T21:47:17.380 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82030.log: /var/log/ceph/cluster1-client.mirror.118497.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60107.log 2026-02-20T21:47:17.380 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82030.log.gz 2026-02-20T21:47:17.380 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.118497.log.gz 2026-02-20T21:47:17.380 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117448.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120954.log 2026-02-20T21:47:17.380 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60107.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.49340.log 2026-02-20T21:47:17.380 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.117448.log.gz 2026-02-20T21:47:17.381 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120954.log: 92.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114552.log 2026-02-20T21:47:17.381 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.120954.log.gz 2026-02-20T21:47:17.382 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127580.log 2026-02-20T21:47:17.382 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.2.49340.log: /var/log/ceph/cluster2-client.mirror.114552.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114552.log.gz 2026-02-20T21:47:17.382 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70547.log 2026-02-20T21:47:17.384 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.127580.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.127580.log.gz 2026-02-20T21:47:17.384 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110868.log 2026-02-20T21:47:17.384 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70547.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70547.log.gz 2026-02-20T21:47:17.385 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65800.log 2026-02-20T21:47:17.385 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110868.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110868.log.gz 2026-02-20T21:47:17.386 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75758.log 2026-02-20T21:47:17.386 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65800.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65800.log.gz 2026-02-20T21:47:17.387 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96624.log 2026-02-20T21:47:17.387 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75758.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75758.log.gz 2026-02-20T21:47:17.387 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73201.log 2026-02-20T21:47:17.388 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96624.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96624.log.gz 2026-02-20T21:47:17.388 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107650.log 2026-02-20T21:47:17.389 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73201.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73201.log.gz 2026-02-20T21:47:17.389 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66416.log 2026-02-20T21:47:17.389 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107650.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107650.log.gz 2026-02-20T21:47:17.390 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138207.log 2026-02-20T21:47:17.390 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66416.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66416.log.gz 2026-02-20T21:47:17.391 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132076.log 2026-02-20T21:47:17.391 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138207.log: 84.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138207.log.gz 2026-02-20T21:47:17.392 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50377.log 2026-02-20T21:47:17.392 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132076.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.132076.log.gz 2026-02-20T21:47:17.392 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105811.log 2026-02-20T21:47:17.392 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50377.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68183.log 2026-02-20T21:47:17.393 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50377.log.gz 2026-02-20T21:47:17.393 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105811.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105811.log.gz 2026-02-20T21:47:17.394 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78996.log 2026-02-20T21:47:17.394 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68183.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68183.log.gz 2026-02-20T21:47:17.395 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83583.log 2026-02-20T21:47:17.395 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78996.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78996.log.gz 2026-02-20T21:47:17.396 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91151.log 2026-02-20T21:47:17.396 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83583.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.83583.log.gz 2026-02-20T21:47:17.396 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93434.log 2026-02-20T21:47:17.405 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91151.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102769.log 2026-02-20T21:47:17.405 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93434.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93434.log.gz 2026-02-20T21:47:17.418 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96082.log 2026-02-20T21:47:17.418 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102769.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102769.log.gz 2026-02-20T21:47:17.431 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77659.log 2026-02-20T21:47:17.431 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96082.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96082.log.gz 2026-02-20T21:47:17.440 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78775.log 2026-02-20T21:47:17.441 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77659.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77659.log.gz 2026-02-20T21:47:17.449 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89938.log.gz 2026-02-20T21:47:17.450 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102180.log 2026-02-20T21:47:17.450 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78775.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78775.log.gz 2026-02-20T21:47:17.451 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70082.log 2026-02-20T21:47:17.452 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102180.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102180.log.gz 2026-02-20T21:47:17.452 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131404.log 2026-02-20T21:47:17.452 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70082.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70082.log.gz 2026-02-20T21:47:17.455 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47961.log 2026-02-20T21:47:17.455 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131404.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131404.log.gz 2026-02-20T21:47:17.455 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138072.log 2026-02-20T21:47:17.455 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.47961.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47961.log.gz 2026-02-20T21:47:17.455 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50155.log 2026-02-20T21:47:17.455 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138072.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138072.log.gz 2026-02-20T21:47:17.455 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67195.log 2026-02-20T21:47:17.455 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50155.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50155.log.gz 2026-02-20T21:47:17.456 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131142.log 2026-02-20T21:47:17.456 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67195.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67195.log.gz 2026-02-20T21:47:17.457 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.127535.log 2026-02-20T21:47:17.457 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131142.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.131142.log.gz 2026-02-20T21:47:17.458 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58177.log 2026-02-20T21:47:17.458 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.127535.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.127535.log.gz 2026-02-20T21:47:17.458 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141923.log 2026-02-20T21:47:17.459 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58177.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58177.log.gz 2026-02-20T21:47:17.459 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.128320.log 2026-02-20T21:47:17.459 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141923.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.141923.log.gz 2026-02-20T21:47:17.460 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115958.log 2026-02-20T21:47:17.461 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.128320.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.128320.log.gz 2026-02-20T21:47:17.462 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103713.log 2026-02-20T21:47:17.462 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115958.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.115958.log.gz 2026-02-20T21:47:17.463 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57436.log 2026-02-20T21:47:17.463 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103713.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.103713.log.gz 2026-02-20T21:47:17.464 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135222.log 2026-02-20T21:47:17.464 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57436.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.57436.log.gz 2026-02-20T21:47:17.465 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68311.log 2026-02-20T21:47:17.466 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135222.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.135222.log.gz 2026-02-20T21:47:17.466 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47850.log 2026-02-20T21:47:17.467 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68311.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68311.log.gz 2026-02-20T21:47:17.467 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137037.log 2026-02-20T21:47:17.469 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.47850.log: 67.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47850.log.gz 2026-02-20T21:47:17.469 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134188.log 2026-02-20T21:47:17.469 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137037.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137037.log.gz 2026-02-20T21:47:17.469 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72377.log 2026-02-20T21:47:17.471 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134188.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134188.log.gz 2026-02-20T21:47:17.471 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64331.log 2026-02-20T21:47:17.471 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.72377.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.admin.72377.log.gz 2026-02-20T21:47:17.471 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94045.log 2026-02-20T21:47:17.472 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64331.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64331.log.gz 2026-02-20T21:47:17.472 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61390.log 2026-02-20T21:47:17.481 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94045.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100207.log 2026-02-20T21:47:17.493 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61390.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61390.log.gz 2026-02-20T21:47:17.494 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138858.log 2026-02-20T21:47:17.495 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100207.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100207.log.gz 2026-02-20T21:47:17.504 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.90517.log.gz 2026-02-20T21:47:17.504 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101170.log 2026-02-20T21:47:17.504 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138858.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.138858.log.gz 2026-02-20T21:47:17.504 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97495.log 2026-02-20T21:47:17.505 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101170.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101170.log.gz 2026-02-20T21:47:17.505 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123222.log 2026-02-20T21:47:17.505 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97495.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97495.log.gz 2026-02-20T21:47:17.505 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121879.log 2026-02-20T21:47:17.508 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123222.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.123222.log.gz 2026-02-20T21:47:17.509 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125691.log 2026-02-20T21:47:17.509 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121879.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.121879.log.gz 2026-02-20T21:47:17.509 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47140.log 2026-02-20T21:47:17.509 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125691.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125691.log.gz 2026-02-20T21:47:17.509 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87063.log 2026-02-20T21:47:17.509 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68369.log 2026-02-20T21:47:17.509 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.47140.log.gz 2026-02-20T21:47:17.515 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87063.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87063.log.gz 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52061.log 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68369.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.68369.log.gz 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78438.log 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52061.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137982.log 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52061.log.gz 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78438.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.78438.log.gz 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127218.log 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137982.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137982.log.gz 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52481.log 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127218.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127218.log.gz 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97309.log 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52481.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71477.log 2026-02-20T21:47:17.516 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52481.log.gz 2026-02-20T21:47:17.518 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97309.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97309.log.gz 2026-02-20T21:47:17.518 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88856.log 2026-02-20T21:47:17.518 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71477.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.141522.log.gz 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71477.log.gz 2026-02-20T21:47:17.518 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:17.518 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112012.log 2026-02-20T21:47:17.519 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66637.log 2026-02-20T21:47:17.519 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88856.log: /var/log/ceph/cluster2-client.mirror.112012.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112012.log.gz 2026-02-20T21:47:17.520 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.143116.log 2026-02-20T21:47:17.520 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66637.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66637.log.gz 2026-02-20T21:47:17.520 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139876.log 2026-02-20T21:47:17.521 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.143116.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.143116.log.gz 2026-02-20T21:47:17.521 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127560.log 2026-02-20T21:47:17.521 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139876.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88856.log.gz 2026-02-20T21:47:17.523 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66695.log 2026-02-20T21:47:17.523 INFO:teuthology.orchestra.run.trial163.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.139876.log.gz 2026-02-20T21:47:17.523 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127560.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67160.log 2026-02-20T21:47:17.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66695.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66695.log.gz 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.127560.log.gz 2026-02-20T21:47:17.525 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:17.525 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133143.log 2026-02-20T21:47:17.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67160.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96802.log 2026-02-20T21:47:17.525 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67160.log.gz 2026-02-20T21:47:17.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133143.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135518.log 2026-02-20T21:47:17.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96802.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96802.log.gz 2026-02-20T21:47:17.527 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.133143.log.gz 2026-02-20T21:47:17.527 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106704.log 2026-02-20T21:47:17.527 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135518.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68555.log 2026-02-20T21:47:17.527 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106704.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106704.log.gz 2026-02-20T21:47:17.527 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127689.log 2026-02-20T21:47:17.527 INFO:teuthology.orchestra.run.trial163.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.135518.log.gz 2026-02-20T21:47:17.531 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71012.log 2026-02-20T21:47:17.531 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127689.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68555.log.gz 2026-02-20T21:47:17.531 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127689.log.gz 2026-02-20T21:47:17.531 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64366.log 2026-02-20T21:47:17.531 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122508.log 2026-02-20T21:47:17.531 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71012.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71012.log.gz 2026-02-20T21:47:17.531 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64366.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.64366.log.gz 2026-02-20T21:47:17.531 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115493.log 2026-02-20T21:47:17.531 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122508.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71663.log 2026-02-20T21:47:17.531 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115493.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115493.log.gz 2026-02-20T21:47:17.532 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141178.log 2026-02-20T21:47:17.533 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71663.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71663.log.gz 2026-02-20T21:47:17.533 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.90632.log 2026-02-20T21:47:17.533 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.141178.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.141178.log.gz 2026-02-20T21:47:17.533 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132596.log 2026-02-20T21:47:17.542 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.1.90632.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137757.log 2026-02-20T21:47:17.555 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132596.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80077.log 2026-02-20T21:47:17.555 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.132596.log.gz 2026-02-20T21:47:17.555 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137757.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.137757.log.gz 2026-02-20T21:47:17.567 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123712.log 2026-02-20T21:47:17.568 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80077.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80077.log.gz 2026-02-20T21:47:17.576 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139277.log 2026-02-20T21:47:17.576 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123712.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123712.log.gz 2026-02-20T21:47:17.585 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.143263.log 2026-02-20T21:47:17.585 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139277.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.139277.log.gz 2026-02-20T21:47:17.599 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73838.log.gz 2026-02-20T21:47:17.600 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123296.log 2026-02-20T21:47:17.600 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.143263.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.143263.log.gz 2026-02-20T21:47:17.600 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64703.log 2026-02-20T21:47:17.609 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123296.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121108.log 2026-02-20T21:47:17.609 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64703.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64703.log.gz 2026-02-20T21:47:17.618 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113752.log 2026-02-20T21:47:17.618 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121108.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121108.log.gz 2026-02-20T21:47:17.638 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89354.log 2026-02-20T21:47:17.646 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113752.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51291.log 2026-02-20T21:47:17.656 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89354.log: 89.4%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55496.log 2026-02-20T21:47:17.656 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.89354.log.gz 2026-02-20T21:47:17.665 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51291.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56722.log 2026-02-20T21:47:17.665 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55496.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55496.log.gz 2026-02-20T21:47:17.668 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51291.log.gz 2026-02-20T21:47:17.673 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97274.log 2026-02-20T21:47:17.682 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56722.log: 88.4% 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.74592.log.gz 2026-02-20T21:47:17.687 INFO:teuthology.orchestra.run.trial163.stderr: 95.7%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.103875.log 2026-02-20T21:47:17.687 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.90852.log.gz 2026-02-20T21:47:17.687 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97274.log: -- replaced with /var/log/ceph/cluster2-client.mirror.56722.log.gz 2026-02-20T21:47:17.687 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97274.log.gz 2026-02-20T21:47:17.687 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129679.log 2026-02-20T21:47:17.691 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.103875.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.103875.log.gz 2026-02-20T21:47:17.691 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49291.log 2026-02-20T21:47:17.691 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129679.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.129679.log.gz 2026-02-20T21:47:17.691 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57145.log 2026-02-20T21:47:17.691 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49291.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94526.log 2026-02-20T21:47:17.699 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97192.log 2026-02-20T21:47:17.703 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94526.log: 84.6%/var/log/ceph/cluster2-client.mirror.57145.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57145.log.gz 2026-02-20T21:47:17.708 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88876.log 2026-02-20T21:47:17.711 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.97192.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.94526.log.gz 2026-02-20T21:47:17.712 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93183.log.gz 2026-02-20T21:47:17.713 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68125.log 2026-02-20T21:47:17.716 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88876.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88876.log.gz 2026-02-20T21:47:17.716 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129386.log 2026-02-20T21:47:17.718 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68125.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68125.log.gz 2026-02-20T21:47:17.718 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121937.log 2026-02-20T21:47:17.734 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129386.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.130381.log.gz 2026-02-20T21:47:17.734 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126030.log 2026-02-20T21:47:17.734 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121937.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121937.log.gz 2026-02-20T21:47:17.734 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.97192.log.gz 2026-02-20T21:47:17.734 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135273.log 2026-02-20T21:47:17.735 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.126030.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.126030.log.gz 2026-02-20T21:47:17.735 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46530.log 2026-02-20T21:47:17.735 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135273.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.135273.log.gz 2026-02-20T21:47:17.735 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114762.log 2026-02-20T21:47:17.735 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.46530.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46530.log.gz 2026-02-20T21:47:17.735 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69520.log 2026-02-20T21:47:17.735 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114762.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114762.log.gz 2026-02-20T21:47:17.735 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139327.log 2026-02-20T21:47:17.735 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69520.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69520.log.gz 2026-02-20T21:47:17.735 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121701.log 2026-02-20T21:47:17.735 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139327.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.139327.log.gz 2026-02-20T21:47:17.736 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54879.log 2026-02-20T21:47:17.736 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121701.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.121701.log.gz 2026-02-20T21:47:17.736 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47918.log 2026-02-20T21:47:17.736 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54879.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54879.log.gz 2026-02-20T21:47:17.736 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138885.log 2026-02-20T21:47:17.736 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.47918.log: 54.5% -- replaced with /var/log/ceph/cluster1-client.mirror.47918.log.gz 2026-02-20T21:47:17.736 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59376.log 2026-02-20T21:47:17.736 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138885.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.138885.log.gz 2026-02-20T21:47:17.736 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123901.log 2026-02-20T21:47:17.736 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59376.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.59376.log.gz 2026-02-20T21:47:17.736 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101255.log 2026-02-20T21:47:17.737 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123901.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123901.log.gz 2026-02-20T21:47:17.737 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89576.log 2026-02-20T21:47:17.737 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101255.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.101255.log.gz 2026-02-20T21:47:17.737 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.129230.log 2026-02-20T21:47:17.737 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.89576.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.89576.log.gz 2026-02-20T21:47:17.737 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134989.log 2026-02-20T21:47:17.743 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.2.129230.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81507.log 2026-02-20T21:47:17.745 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134989.log: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.2.129230.log.gz 2026-02-20T21:47:17.746 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86795.log 2026-02-20T21:47:17.750 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81507.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.81507.log.gz 2026-02-20T21:47:17.750 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78589.log 2026-02-20T21:47:17.750 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86795.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86795.log.gz 2026-02-20T21:47:17.750 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139697.log 2026-02-20T21:47:17.750 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78589.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78589.log.gz 2026-02-20T21:47:17.750 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135293.log 2026-02-20T21:47:17.750 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139697.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.139697.log.gz 2026-02-20T21:47:17.751 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125555.log 2026-02-20T21:47:17.759 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135293.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.135293.log.gz 2026-02-20T21:47:17.760 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102715.log 2026-02-20T21:47:17.760 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125555.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125555.log.gz 2026-02-20T21:47:17.760 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134969.log 2026-02-20T21:47:17.760 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102715.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.102715.log.gz 2026-02-20T21:47:17.760 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108145.log 2026-02-20T21:47:17.760 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134969.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.134969.log.gz 2026-02-20T21:47:17.760 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64145.log 2026-02-20T21:47:17.760 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108145.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108145.log.gz 2026-02-20T21:47:17.760 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135732.log 2026-02-20T21:47:17.760 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64145.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64145.log.gz 2026-02-20T21:47:17.760 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75700.log 2026-02-20T21:47:17.761 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135732.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.134989.log.gz 2026-02-20T21:47:17.761 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.135732.log.gz 2026-02-20T21:47:17.761 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139483.log 2026-02-20T21:47:17.761 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75700.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75700.log.gz 2026-02-20T21:47:17.761 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87462.log 2026-02-20T21:47:17.761 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139483.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.139483.log.gz 2026-02-20T21:47:17.761 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54584.log 2026-02-20T21:47:17.764 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.87462.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.87462.log.gz 2026-02-20T21:47:17.764 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89722.log 2026-02-20T21:47:17.764 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54584.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54584.log.gz 2026-02-20T21:47:17.764 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134279.log 2026-02-20T21:47:17.773 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89722.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133371.log 2026-02-20T21:47:17.786 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134279.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.134279.log.gz 2026-02-20T21:47:17.788 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91151.log.gz 2026-02-20T21:47:17.791 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46775.log 2026-02-20T21:47:17.791 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133371.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133371.log.gz 2026-02-20T21:47:17.791 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92881.log 2026-02-20T21:47:17.791 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.46775.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46775.log.gz 2026-02-20T21:47:17.791 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114924.log 2026-02-20T21:47:17.794 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92881.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.97679.log.gz 2026-02-20T21:47:17.797 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.122508.log.gz 2026-02-20T21:47:17.799 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137959.log 2026-02-20T21:47:17.811 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62006.log 2026-02-20T21:47:17.812 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114924.log: /var/log/ceph/cluster1-client.mirror.137959.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.114924.log.gz 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.137959.log.gz 2026-02-20T21:47:17.812 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:17.812 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57048.log 2026-02-20T21:47:17.812 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62006.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139791.log 2026-02-20T21:47:17.812 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62006.log.gz 2026-02-20T21:47:17.812 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57886.log 2026-02-20T21:47:17.812 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139791.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57048.log.gz 2026-02-20T21:47:17.812 INFO:teuthology.orchestra.run.trial163.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.139791.log.gz 2026-02-20T21:47:17.813 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53529.log 2026-02-20T21:47:17.813 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57886.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54177.log 2026-02-20T21:47:17.813 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57886.log.gz 2026-02-20T21:47:17.813 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53529.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88164.log 2026-02-20T21:47:17.813 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.54177.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53529.log.gz 2026-02-20T21:47:17.813 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.54177.log.gz 2026-02-20T21:47:17.813 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131212.log 2026-02-20T21:47:17.813 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81193.log 2026-02-20T21:47:17.813 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88164.log: /var/log/ceph/cluster2-client.mirror.131212.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131212.log.gz 2026-02-20T21:47:17.813 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74301.log 2026-02-20T21:47:17.813 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81193.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81193.log.gz 2026-02-20T21:47:17.813 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74735.log 2026-02-20T21:47:17.814 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74301.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74301.log.gz 2026-02-20T21:47:17.814 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74677.log 2026-02-20T21:47:17.814 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74735.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74735.log.gz 2026-02-20T21:47:17.814 INFO:teuthology.orchestra.run.trial163.stderr: 92.7%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106974.log 2026-02-20T21:47:17.814 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.88164.log.gz 2026-02-20T21:47:17.814 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74677.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93722.log 2026-02-20T21:47:17.814 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106974.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74677.log.gz 2026-02-20T21:47:17.814 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120501.log 2026-02-20T21:47:17.814 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93722.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.106974.log.gz 2026-02-20T21:47:17.814 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111977.log 2026-02-20T21:47:17.814 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120501.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120501.log.gz 2026-02-20T21:47:17.821 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88360.log 2026-02-20T21:47:17.821 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111977.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.111977.log.gz 2026-02-20T21:47:17.821 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84771.log 2026-02-20T21:47:17.825 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88360.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88360.log.gz 2026-02-20T21:47:17.825 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74828.log 2026-02-20T21:47:17.825 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84771.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.84771.log.gz 2026-02-20T21:47:17.831 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119123.log 2026-02-20T21:47:17.832 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74828.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74828.log.gz 2026-02-20T21:47:17.832 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55558.log 2026-02-20T21:47:17.832 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119123.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119123.log.gz 2026-02-20T21:47:17.832 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124560.log 2026-02-20T21:47:17.832 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55558.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55558.log.gz 2026-02-20T21:47:17.832 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51112.log 2026-02-20T21:47:17.832 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124560.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124560.log.gz 2026-02-20T21:47:17.832 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121733.log 2026-02-20T21:47:17.832 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51112.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51112.log.gz 2026-02-20T21:47:17.832 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56660.log 2026-02-20T21:47:17.832 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.121733.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.121733.log.gz 2026-02-20T21:47:17.832 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114458.log 2026-02-20T21:47:17.834 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56660.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56660.log.gz 2026-02-20T21:47:17.835 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134166.log 2026-02-20T21:47:17.835 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114458.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.114458.log.gz 2026-02-20T21:47:17.835 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68218.log 2026-02-20T21:47:17.838 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134166.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.134166.log.gz 2026-02-20T21:47:17.838 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102021.log 2026-02-20T21:47:17.838 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68218.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68218.log.gz 2026-02-20T21:47:17.838 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140726.log 2026-02-20T21:47:17.838 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102021.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102021.log.gz 2026-02-20T21:47:17.838 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58662.log 2026-02-20T21:47:17.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140726.log: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.140726.log.gz 2026-02-20T21:47:17.840 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84670.log 2026-02-20T21:47:17.840 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58662.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58662.log.gz 2026-02-20T21:47:17.842 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110219.log 2026-02-20T21:47:17.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84670.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.84670.log.gz 2026-02-20T21:47:17.842 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135640.log 2026-02-20T21:47:17.842 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110219.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.110219.log.gz 2026-02-20T21:47:17.846 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139373.log 2026-02-20T21:47:17.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135640.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.135640.log.gz 2026-02-20T21:47:17.846 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140437.log 2026-02-20T21:47:17.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139373.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.139373.log.gz 2026-02-20T21:47:17.846 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137937.log 2026-02-20T21:47:17.846 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140437.log: 82.2% -- replaced with /var/log/ceph/cluster2-client.mirror.140437.log.gz 2026-02-20T21:47:17.846 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89287.log 2026-02-20T21:47:17.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137937.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137937.log.gz 2026-02-20T21:47:17.847 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125073.log 2026-02-20T21:47:17.850 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89287.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.89287.log.gz 2026-02-20T21:47:17.850 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50902.log 2026-02-20T21:47:17.850 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125073.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.125073.log.gz 2026-02-20T21:47:17.850 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.47225.log 2026-02-20T21:47:17.850 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50902.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.126075.log 2026-02-20T21:47:17.850 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.50902.log.gz 2026-02-20T21:47:17.850 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.47225.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105414.log 2026-02-20T21:47:17.853 INFO:teuthology.orchestra.run.trial163.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.47225.log.gz 2026-02-20T21:47:17.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.126075.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.126075.log.gz 2026-02-20T21:47:17.856 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132041.log 2026-02-20T21:47:17.856 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105414.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105414.log.gz 2026-02-20T21:47:17.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137779.log 2026-02-20T21:47:17.858 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132041.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.132041.log.gz 2026-02-20T21:47:17.858 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76021.log 2026-02-20T21:47:17.858 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137779.log: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.137779.log.gz 2026-02-20T21:47:17.859 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92152.log 2026-02-20T21:47:17.859 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76021.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76021.log.gz 2026-02-20T21:47:17.859 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57727.log 2026-02-20T21:47:17.868 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92152.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110456.log 2026-02-20T21:47:17.869 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57727.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57727.log.gz 2026-02-20T21:47:17.882 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102626.log 2026-02-20T21:47:17.882 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110456.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110456.log.gz 2026-02-20T21:47:17.890 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.47140.log 2026-02-20T21:47:17.891 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102626.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102626.log.gz 2026-02-20T21:47:17.905 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64238.log 2026-02-20T21:47:17.915 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.0.47140.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123388.log 2026-02-20T21:47:17.915 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64238.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64238.log.gz 2026-02-20T21:47:17.928 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66230.log 2026-02-20T21:47:17.928 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123388.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123388.log.gz 2026-02-20T21:47:17.938 INFO:teuthology.orchestra.run.trial163.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.60107.log.gz 2026-02-20T21:47:17.938 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89459.log 2026-02-20T21:47:17.938 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66230.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142106.log 2026-02-20T21:47:17.938 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.66230.log.gz 2026-02-20T21:47:17.939 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89459.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89459.log.gz 2026-02-20T21:47:17.955 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62378.log 2026-02-20T21:47:17.956 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142106.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.142106.log.gz 2026-02-20T21:47:17.964 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78159.log 2026-02-20T21:47:17.976 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62378.log: 88.6% 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.123296.log.gz 2026-02-20T21:47:17.976 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.62378.log.gz 2026-02-20T21:47:17.977 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72277.log 2026-02-20T21:47:17.977 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78159.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78159.log.gz 2026-02-20T21:47:17.977 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103242.log 2026-02-20T21:47:17.981 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.72277.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.72277.log.gz 2026-02-20T21:47:17.981 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66974.log 2026-02-20T21:47:17.981 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103242.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.103242.log.gz 2026-02-20T21:47:17.981 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73669.log 2026-02-20T21:47:17.981 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66974.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66974.log.gz 2026-02-20T21:47:17.983 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139350.log 2026-02-20T21:47:17.991 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.73669.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73669.log.gz 2026-02-20T21:47:17.991 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65321.log 2026-02-20T21:47:17.991 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139350.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.139350.log.gz 2026-02-20T21:47:17.991 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52236.log 2026-02-20T21:47:17.991 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.65321.log: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.65321.log.gz 2026-02-20T21:47:17.991 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116253.log 2026-02-20T21:47:17.991 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52236.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52236.log.gz 2026-02-20T21:47:17.991 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59861.log 2026-02-20T21:47:17.991 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116253.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.116253.log.gz 2026-02-20T21:47:17.991 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99109.log 2026-02-20T21:47:17.992 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59861.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59861.log.gz 2026-02-20T21:47:17.992 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109033.log 2026-02-20T21:47:18.001 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99109.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90007.log.gz 2026-02-20T21:47:18.001 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100349.log 2026-02-20T21:47:18.001 INFO:teuthology.orchestra.run.trial163.stderr:gzip/var/log/ceph/cluster1-client.mirror.109033.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89501.log 2026-02-20T21:47:18.002 INFO:teuthology.orchestra.run.trial163.stderr: 88.1%/var/log/ceph/cluster2-client.mirror.100349.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.100349.log.gz 2026-02-20T21:47:18.002 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135589.log 2026-02-20T21:47:18.003 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89501.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89501.log.gz 2026-02-20T21:47:18.003 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59535.log 2026-02-20T21:47:18.007 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135589.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.135589.log.gz 2026-02-20T21:47:18.007 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60890.log 2026-02-20T21:47:18.007 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59535.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.59535.log.gz 2026-02-20T21:47:18.007 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88727.log 2026-02-20T21:47:18.007 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60890.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60890.log.gz 2026-02-20T21:47:18.007 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55787.log 2026-02-20T21:47:18.013 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88727.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88727.log.gz 2026-02-20T21:47:18.013 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117729.log 2026-02-20T21:47:18.013 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55787.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55787.log.gz 2026-02-20T21:47:18.013 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57630.log 2026-02-20T21:47:18.013 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117729.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.117729.log.gz 2026-02-20T21:47:18.013 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.109033.log.gz 2026-02-20T21:47:18.013 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99042.log 2026-02-20T21:47:18.027 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57630.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.57630.log.gz 2026-02-20T21:47:18.028 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.142608.log 2026-02-20T21:47:18.028 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.99042.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.99042.log.gz 2026-02-20T21:47:18.028 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66172.log 2026-02-20T21:47:18.028 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.142608.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.142608.log.gz 2026-02-20T21:47:18.028 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91961.log 2026-02-20T21:47:18.028 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66172.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.66172.log.gz 2026-02-20T21:47:18.028 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63994.log 2026-02-20T21:47:18.028 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91961.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91961.log.gz 2026-02-20T21:47:18.028 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82534.log 2026-02-20T21:47:18.028 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63994.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63994.log.gz 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56819.log 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82534.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82534.log.gz 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46094.log 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56819.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56819.log.gz 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67939.log 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.46094.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46094.log.gz 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67288.log 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67939.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67939.log.gz 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131107.log 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67288.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67288.log.gz 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55752.log 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131107.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131107.log.gz 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96132.log 2026-02-20T21:47:18.029 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55752.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55752.log.gz 2026-02-20T21:47:18.030 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91098.log 2026-02-20T21:47:18.030 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96132.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96132.log.gz 2026-02-20T21:47:18.030 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.141258.log 2026-02-20T21:47:18.030 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91098.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.89647.log.gz 2026-02-20T21:47:18.032 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82344.log 2026-02-20T21:47:18.032 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.0.141258.log: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.0.141258.log.gz 2026-02-20T21:47:18.032 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89198.log 2026-02-20T21:47:18.032 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82344.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.82344.log.gz 2026-02-20T21:47:18.032 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46240.log 2026-02-20T21:47:18.034 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.89198.log: 94.7% -- replaced with /var/log/ceph/cluster1-client.mirror.89198.log.gz 2026-02-20T21:47:18.035 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135018.log 2026-02-20T21:47:18.035 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.46240.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46240.log.gz 2026-02-20T21:47:18.035 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64738.log 2026-02-20T21:47:18.036 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135018.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.135018.log.gz 2026-02-20T21:47:18.036 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109719.log 2026-02-20T21:47:18.036 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64738.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64738.log.gz 2026-02-20T21:47:18.038 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89523.log 2026-02-20T21:47:18.038 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109719.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.109719.log.gz 2026-02-20T21:47:18.038 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137599.log 2026-02-20T21:47:18.038 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89523.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89523.log.gz 2026-02-20T21:47:18.040 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141412.log 2026-02-20T21:47:18.041 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137599.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137599.log.gz 2026-02-20T21:47:18.041 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60704.log 2026-02-20T21:47:18.041 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141412.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.141412.log.gz 2026-02-20T21:47:18.041 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81786.log 2026-02-20T21:47:18.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60704.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60704.log.gz 2026-02-20T21:47:18.048 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105722.log 2026-02-20T21:47:18.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81786.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81786.log.gz 2026-02-20T21:47:18.048 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124139.log 2026-02-20T21:47:18.048 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105722.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105722.log.gz 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104500.log 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.94045.log.gz 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124139.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90544.log 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%/var/log/ceph/cluster2-client.mirror.104500.log: -- replaced with /var/log/ceph/cluster2-client.mirror.124139.log.gz 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75049.log 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.104500.log.gz 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90544.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100511.log 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75049.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.90544.log.gz 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75049.log.gz 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116367.log 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.100511.log: -- /var/log/ceph/cluster2-client.mirror.78252.log 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.100511.log.gz 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116367.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.116367.log.gz 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65236.log 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.113752.log.gz 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78252.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120331.log 2026-02-20T21:47:18.049 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.78252.log.gz 2026-02-20T21:47:18.050 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88340.log 2026-02-20T21:47:18.050 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65236.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114843.log 2026-02-20T21:47:18.050 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120331.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120331.log.gz/var/log/ceph/cluster2-client.mirror.88340.log: 2026-02-20T21:47:18.055 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74863.log 2026-02-20T21:47:18.055 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.114843.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.114843.log.gz 2026-02-20T21:47:18.055 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96554.log 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74863.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.74863.log.gz 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77880.log 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65259.log.gz 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96554.log: 88.0%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55593.log 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.96554.log.gz 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77880.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65951.log 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77880.log.gz 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55593.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134024.log 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65951.log: 88.5% 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88340.log.gz 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.55593.log.gz 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65951.log.gz 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118004.log 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134024.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136161.log 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118004.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120974.log 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.134024.log.gz 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.118004.log.gz 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136161.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90153.log 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%/var/log/ceph/cluster1-client.mirror.120974.log: -- replaced with /var/log/ceph/cluster2-client.mirror.136161.log.gz 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96167.log 2026-02-20T21:47:18.056 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90153.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.120974.log.gz 2026-02-20T21:47:18.057 INFO:teuthology.orchestra.run.trial163.stderr: 97.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65236.log.gz 2026-02-20T21:47:18.057 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60925.log 2026-02-20T21:47:18.060 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96167.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127832.log 2026-02-20T21:47:18.060 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60925.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96167.log.gzgzip 2026-02-20T21:47:18.060 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125963.log 2026-02-20T21:47:18.060 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.60925.log.gz 2026-02-20T21:47:18.060 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127832.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88833.log 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123747.log 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125963.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127832.log.gz 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr: 88.2%/var/log/ceph/cluster1-client.mirror.88833.log: -- replaced with /var/log/ceph/cluster2-client.mirror.125963.log.gz 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55108.log 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100091.log 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123747.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123747.log.gz 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55108.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55108.log.gz 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63308.log 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100091.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100091.log.gz 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124814.log 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63308.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115458.log 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63308.log.gz 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr: 93.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88833.log.gz 2026-02-20T21:47:18.061 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124814.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72346.log 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115458.log: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111141.log 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.124814.log.gz 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115458.log.gz 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.72346.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102858.log 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.72346.log.gz/var/log/ceph/cluster2-client.mirror.111141.log: 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133442.log 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.111141.log.gz 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83695.log 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.129386.log.gz 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102858.log: /var/log/ceph/cluster2-client.mirror.133442.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102858.log.gz 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62750.log 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.133442.log.gz 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.94146.log 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83695.log: /var/log/ceph/cluster2-client.mirror.62750.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71814.log 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83695.log.gz 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.2.94146.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62750.log.gz 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54390.log 2026-02-20T21:47:18.065 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71814.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133762.log 2026-02-20T21:47:18.066 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54390.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107248.log 2026-02-20T21:47:18.066 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71814.log.gz 2026-02-20T21:47:18.066 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133762.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54390.log.gz 2026-02-20T21:47:18.066 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133762.log.gz 2026-02-20T21:47:18.066 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65986.log 2026-02-20T21:47:18.067 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107248.log: gzip 88.2% -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133782.log 2026-02-20T21:47:18.067 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.107248.log.gz 2026-02-20T21:47:18.067 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65986.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68741.log 2026-02-20T21:47:18.067 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.65986.log.gz 2026-02-20T21:47:18.067 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133782.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99169.log 2026-02-20T21:47:18.067 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68741.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82906.log 2026-02-20T21:47:18.067 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68741.log.gz 2026-02-20T21:47:18.068 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99169.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67067.log 2026-02-20T21:47:18.068 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.99169.log.gz 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.133782.log.gz 2026-02-20T21:47:18.068 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82906.log: 2026-02-20T21:47:18.069 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82906.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103855.log 2026-02-20T21:47:18.069 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:18.069 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67067.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.114878.log 2026-02-20T21:47:18.069 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.67067.log.gz 2026-02-20T21:47:18.069 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60413.log 2026-02-20T21:47:18.069 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103855.log: /var/log/ceph/cluster2-client.mirror.114878.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73649.log 2026-02-20T21:47:18.069 INFO:teuthology.orchestra.run.trial163.stderr: 88.3%/var/log/ceph/cluster1-client.admin.60413.log: -- replaced with /var/log/ceph/cluster2-client.mirror.114878.log.gz 2026-02-20T21:47:18.069 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.admin.60413.log.gz 2026-02-20T21:47:18.070 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127370.log 2026-02-20T21:47:18.070 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73649.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136722.log 2026-02-20T21:47:18.070 INFO:teuthology.orchestra.run.trial163.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.103855.log.gz 2026-02-20T21:47:18.070 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127370.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127370.log.gz 2026-02-20T21:47:18.070 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98975.log 2026-02-20T21:47:18.071 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127994.log 2026-02-20T21:47:18.071 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136722.log: /var/log/ceph/cluster2-client.mirror.98975.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.136722.log.gz 2026-02-20T21:47:18.071 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98975.log.gz 2026-02-20T21:47:18.071 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67660.log 2026-02-20T21:47:18.072 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80263.log 2026-02-20T21:47:18.072 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127994.log: /var/log/ceph/cluster2-client.mirror.67660.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.127994.log.gz 2026-02-20T21:47:18.072 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67660.log.gz 2026-02-20T21:47:18.072 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81658.log 2026-02-20T21:47:18.072 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.141685.log 2026-02-20T21:47:18.073 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80263.log: /var/log/ceph/cluster2-client.mirror.81658.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.73649.log.gz 2026-02-20T21:47:18.073 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80263.log.gz 2026-02-20T21:47:18.073 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81658.log.gz 2026-02-20T21:47:18.073 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42682.log 2026-02-20T21:47:18.074 INFO:teuthology.orchestra.run.trial163.stderr:gzip/var/log/ceph/cluster2-client.mirror.2.141685.log: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53610.log 2026-02-20T21:47:18.074 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.42682.log: 0.0%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129230.log 2026-02-20T21:47:18.074 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.42682.log.gz 2026-02-20T21:47:18.075 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53610.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53610.log.gz 2026-02-20T21:47:18.075 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105900.log 2026-02-20T21:47:18.075 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77136.log 2026-02-20T21:47:18.075 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129230.log: 92.3%/var/log/ceph/cluster2-client.mirror.105900.log: -- replaced with /var/log/ceph/cluster2-client.mirror.2.141685.log.gz 2026-02-20T21:47:18.077 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105900.log.gz 2026-02-20T21:47:18.077 INFO:teuthology.orchestra.run.trial163.stderr: 93.1% -- replaced with /var/log/ceph/cluster1-client.mirror.129230.log.gz 2026-02-20T21:47:18.077 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139181.log 2026-02-20T21:47:18.077 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77136.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135809.log 2026-02-20T21:47:18.077 INFO:teuthology.orchestra.run.trial163.stderr: 89.0%/var/log/ceph/cluster2-client.mirror.139181.log: -- replaced with /var/log/ceph/cluster2-client.mirror.77136.log.gz 2026-02-20T21:47:18.077 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78031.log 2026-02-20T21:47:18.077 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.139181.log.gz 2026-02-20T21:47:18.077 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135809.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55690.log 2026-02-20T21:47:18.078 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78031.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95801.log 2026-02-20T21:47:18.078 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55690.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78031.log.gz 2026-02-20T21:47:18.078 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.92831.log.gz 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.135809.log.gz 2026-02-20T21:47:18.078 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:18.078 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55690.log.gz 2026-02-20T21:47:18.078 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125038.log 2026-02-20T21:47:18.079 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95801.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71070.log 2026-02-20T21:47:18.079 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125038.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89090.log 2026-02-20T21:47:18.079 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71070.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120784.log 2026-02-20T21:47:18.079 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.125038.log.gz 2026-02-20T21:47:18.080 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71070.log.gz 2026-02-20T21:47:18.080 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89090.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132284.log 2026-02-20T21:47:18.080 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120784.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120784.log.gz 2026-02-20T21:47:18.080 INFO:teuthology.orchestra.run.trial163.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89090.log.gz 2026-02-20T21:47:18.081 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96252.log 2026-02-20T21:47:18.081 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132284.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58406.log 2026-02-20T21:47:18.081 INFO:teuthology.orchestra.run.trial163.stderr: 87.9%/var/log/ceph/cluster2-client.mirror.96252.log: -- replaced with /var/log/ceph/cluster2-client.mirror.132284.log.gz 2026-02-20T21:47:18.081 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119208.log 2026-02-20T21:47:18.081 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96252.log.gz 2026-02-20T21:47:18.081 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108675.log 2026-02-20T21:47:18.081 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58406.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58406.log.gz 2026-02-20T21:47:18.081 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119208.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119208.log.gz 2026-02-20T21:47:18.082 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.123978.log 2026-02-20T21:47:18.082 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108675.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124201.log 2026-02-20T21:47:18.082 INFO:teuthology.orchestra.run.trial163.stderr: 88.1%/var/log/ceph/cluster1-client.mirror.123978.log: -- replaced with /var/log/ceph/cluster2-client.mirror.108675.log.gz 2026-02-20T21:47:18.082 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57110.log 2026-02-20T21:47:18.083 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.124201.log: /var/log/ceph/cluster1-client.mirror.137464.log 2026-02-20T21:47:18.083 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124201.log.gz 2026-02-20T21:47:18.083 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57110.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57110.log.gz 2026-02-20T21:47:18.083 INFO:teuthology.orchestra.run.trial163.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.123978.log.gz 2026-02-20T21:47:18.088 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55849.log 2026-02-20T21:47:18.088 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137464.log: gzip -5 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137464.log.gz 2026-02-20T21:47:18.088 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.141861.log 2026-02-20T21:47:18.088 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55849.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127867.log 2026-02-20T21:47:18.088 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55849.log.gz 2026-02-20T21:47:18.088 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141861.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121390.log 2026-02-20T21:47:18.088 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112413.log 2026-02-20T21:47:18.089 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127867.log: /var/log/ceph/cluster2-client.mirror.121390.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127867.log.gz 2026-02-20T21:47:18.089 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.121390.log.gz 2026-02-20T21:47:18.089 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60983.log 2026-02-20T21:47:18.090 INFO:teuthology.orchestra.run.trial163.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.141861.log.gz 2026-02-20T21:47:18.090 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.47400.log 2026-02-20T21:47:18.090 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112413.log: /var/log/ceph/cluster2-client.mirror.60983.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73734.log 2026-02-20T21:47:18.090 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112413.log.gz 2026-02-20T21:47:18.090 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.2.47400.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.60983.log.gz 2026-02-20T21:47:18.090 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127751.log 2026-02-20T21:47:18.090 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138094.log 2026-02-20T21:47:18.091 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73734.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.99109.log.gz 2026-02-20T21:47:18.091 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127751.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.127751.log.gz 2026-02-20T21:47:18.091 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91303.log 2026-02-20T21:47:18.091 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90494.log 2026-02-20T21:47:18.092 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138094.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138094.log.gz 2026-02-20T21:47:18.092 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91303.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88985.log 2026-02-20T21:47:18.092 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90494.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.90494.log.gz 2026-02-20T21:47:18.093 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54487.log 2026-02-20T21:47:18.093 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88985.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88985.log.gz 2026-02-20T21:47:18.094 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66323.log 2026-02-20T21:47:18.094 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54487.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54487.log.gz 2026-02-20T21:47:18.095 INFO:teuthology.orchestra.run.trial163.stderr: 96.4% -- replaced with /var/log/ceph/cluster2-client.mirror.73734.log.gz 2026-02-20T21:47:18.095 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97545.log 2026-02-20T21:47:18.095 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66323.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100314.log 2026-02-20T21:47:18.095 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66323.log.gz 2026-02-20T21:47:18.096 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97545.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90567.log 2026-02-20T21:47:18.096 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100314.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.2.47400.log.gz 2026-02-20T21:47:18.096 INFO:teuthology.orchestra.run.trial163.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.97545.log.gz 2026-02-20T21:47:18.096 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100314.log.gz 2026-02-20T21:47:18.097 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119689.log 2026-02-20T21:47:18.097 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90567.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88555.log 2026-02-20T21:47:18.097 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119689.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.47225.log 2026-02-20T21:47:18.101 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88555.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88555.log.gz 2026-02-20T21:47:18.102 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105099.log 2026-02-20T21:47:18.105 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.2.47225.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134450.log 2026-02-20T21:47:18.117 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105099.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.105099.log.gz 2026-02-20T21:47:18.121 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55981.log 2026-02-20T21:47:18.122 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134450.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134450.log.gz 2026-02-20T21:47:18.128 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99610.log 2026-02-20T21:47:18.128 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55981.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55981.log.gz 2026-02-20T21:47:18.141 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81100.log 2026-02-20T21:47:18.141 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99610.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99610.log.gz 2026-02-20T21:47:18.148 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82720.log 2026-02-20T21:47:18.161 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81100.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81100.log.gz 2026-02-20T21:47:18.161 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58018.log 2026-02-20T21:47:18.162 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82720.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82720.log.gz 2026-02-20T21:47:18.169 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66916.log 2026-02-20T21:47:18.169 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58018.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58018.log.gz 2026-02-20T21:47:18.173 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49146.log.gz 2026-02-20T21:47:18.175 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.119689.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49630.log 2026-02-20T21:47:18.175 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:18.175 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66916.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49340.log 2026-02-20T21:47:18.175 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66916.log.gz 2026-02-20T21:47:18.175 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49630.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48328.log 2026-02-20T21:47:18.175 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49630.log.gz 2026-02-20T21:47:18.175 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49340.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94767.log 2026-02-20T21:47:18.177 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48328.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48328.log.gz 2026-02-20T21:47:18.177 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139229.log 2026-02-20T21:47:18.186 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94767.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68648.log 2026-02-20T21:47:18.189 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139229.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.49340.log.gz 2026-02-20T21:47:18.194 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121576.log 2026-02-20T21:47:18.196 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68648.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120897.log 2026-02-20T21:47:18.196 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121576.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121576.log.gz 2026-02-20T21:47:18.197 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138387.log 2026-02-20T21:47:18.197 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120897.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120897.log.gz 2026-02-20T21:47:18.200 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48345.log 2026-02-20T21:47:18.201 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138387.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138387.log.gz 2026-02-20T21:47:18.201 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99440.log 2026-02-20T21:47:18.201 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48345.log: 4.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48345.log.gz 2026-02-20T21:47:18.201 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139433.log 2026-02-20T21:47:18.201 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99440.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.99440.log.gz 2026-02-20T21:47:18.201 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119158.log 2026-02-20T21:47:18.207 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139433.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68648.log.gz 2026-02-20T21:47:18.213 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110065.log 2026-02-20T21:47:18.214 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119158.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119158.log.gz 2026-02-20T21:47:18.235 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123165.log 2026-02-20T21:47:18.236 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110065.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110065.log.gz 2026-02-20T21:47:18.247 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129319.log 2026-02-20T21:47:18.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123165.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.123165.log.gz 2026-02-20T21:47:18.258 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73981.log 2026-02-20T21:47:18.259 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129319.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.129319.log.gz 2026-02-20T21:47:18.270 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83185.log 2026-02-20T21:47:18.271 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73981.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73981.log.gz 2026-02-20T21:47:18.278 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.140277.log.gz 2026-02-20T21:47:18.281 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138027.log 2026-02-20T21:47:18.281 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83185.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.83185.log.gz 2026-02-20T21:47:18.281 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135060.log 2026-02-20T21:47:18.281 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138027.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138027.log.gz 2026-02-20T21:47:18.281 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.140437.log 2026-02-20T21:47:18.281 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135060.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.135060.log.gz 2026-02-20T21:47:18.282 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70047.log 2026-02-20T21:47:18.291 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.3.140437.log: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.3.140437.log.gz 2026-02-20T21:47:18.291 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80856.log 2026-02-20T21:47:18.291 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70047.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70047.log.gz 2026-02-20T21:47:18.292 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142168.log 2026-02-20T21:47:18.292 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80856.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80856.log.gz 2026-02-20T21:47:18.292 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112967.log 2026-02-20T21:47:18.292 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142168.log: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.142168.log.gz 2026-02-20T21:47:18.292 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111176.log 2026-02-20T21:47:18.300 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112967.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62936.log 2026-02-20T21:47:18.301 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111176.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.111176.log.gz 2026-02-20T21:47:18.318 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126141.log 2026-02-20T21:47:18.319 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62936.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62936.log.gz 2026-02-20T21:47:18.319 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136699.log 2026-02-20T21:47:18.319 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126141.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.126141.log.gz 2026-02-20T21:47:18.319 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.142493.log 2026-02-20T21:47:18.326 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136699.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.136699.log.gz 2026-02-20T21:47:18.328 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70698.log 2026-02-20T21:47:18.334 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.142493.log: 86.6% 90.9% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-20T21:47:18.337 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95962.log 2026-02-20T21:47:18.337 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70698.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70698.log.gz 2026-02-20T21:47:18.337 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.142493.log.gz 2026-02-20T21:47:18.338 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77008.log 2026-02-20T21:47:18.338 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95962.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95962.log.gz 2026-02-20T21:47:18.338 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108446.log 2026-02-20T21:47:18.338 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77008.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77008.log.gz 2026-02-20T21:47:18.338 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108729.log 2026-02-20T21:47:18.341 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108446.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.108446.log.gz 2026-02-20T21:47:18.341 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89984.log 2026-02-20T21:47:18.342 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108729.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108729.log.gz 2026-02-20T21:47:18.342 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54720.log 2026-02-20T21:47:18.342 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89984.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89984.log.gz 2026-02-20T21:47:18.342 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88575.log 2026-02-20T21:47:18.342 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54720.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54720.log.gz 2026-02-20T21:47:18.342 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.143018.log 2026-02-20T21:47:18.355 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88575.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88575.log.gz 2026-02-20T21:47:18.356 INFO:teuthology.orchestra.run.trial163.stderr: 95.4% -- replaced with /var/log/ceph/cluster2-client.mirror.92152.log.gz 2026-02-20T21:47:18.356 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61762.log 2026-02-20T21:47:18.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.143018.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.92881.log.gz 2026-02-20T21:47:18.356 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.143018.log.gz 2026-02-20T21:47:18.356 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105865.log 2026-02-20T21:47:18.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61762.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75479.log 2026-02-20T21:47:18.356 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61762.log.gz 2026-02-20T21:47:18.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105865.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105865.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134685.log 2026-02-20T21:47:18.356 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:18.356 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75479.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.75479.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124444.log 2026-02-20T21:47:18.356 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:18.357 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134685.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110254.log 2026-02-20T21:47:18.357 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.134685.log.gz 2026-02-20T21:47:18.357 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124444.log: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99087.log 2026-02-20T21:47:18.357 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.124444.log.gz 2026-02-20T21:47:18.357 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110254.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.110254.log.gz 2026-02-20T21:47:18.357 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50260.log 2026-02-20T21:47:18.357 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134492.log 2026-02-20T21:47:18.357 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.99087.log: /var/log/ceph/cluster2-client.mirror.50260.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50260.log.gz 2026-02-20T21:47:18.357 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57692.log 2026-02-20T21:47:18.357 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134492.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.99087.log.gz 2026-02-20T21:47:18.357 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92180.log 2026-02-20T21:47:18.357 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57692.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.134492.log.gz 2026-02-20T21:47:18.358 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57692.log.gz 2026-02-20T21:47:18.358 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88250.log 2026-02-20T21:47:18.358 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92180.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80205.log 2026-02-20T21:47:18.358 INFO:teuthology.orchestra.run.trial163.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.92180.log.gz 2026-02-20T21:47:18.358 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88250.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.108355.log 2026-02-20T21:47:18.358 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80205.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.80205.log.gz 2026-02-20T21:47:18.358 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.143508.log 2026-02-20T21:47:18.366 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.108355.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.108355.log.gz 2026-02-20T21:47:18.366 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132238.log 2026-02-20T21:47:18.366 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.143508.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.143508.log.gz 2026-02-20T21:47:18.366 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.88250.log.gz 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr: -- /var/log/ceph/cluster2-client.mirror.89006.log 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112566.log 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132238.log: /var/log/ceph/cluster2-client.mirror.89006.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.132238.log.gz 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.89006.log.gz 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75572.log 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112566.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47731.log 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75572.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112566.log.gz 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75572.log.gz 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108278.log 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.47731.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136064.log 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.admin.47731.log.gz 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108278.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83937.log 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136064.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.108278.log.gz 2026-02-20T21:47:18.367 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.136064.log.gz 2026-02-20T21:47:18.368 INFO:teuthology.orchestra.run.trial163.stderr: /var/log/ceph/cluster2-client.mirror.82499.log 2026-02-20T21:47:18.368 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95497.log 2026-02-20T21:47:18.368 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83937.log: /var/log/ceph/cluster2-client.mirror.82499.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82499.log.gz 2026-02-20T21:47:18.368 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96217.log 2026-02-20T21:47:18.368 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95497.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95497.log.gz 2026-02-20T21:47:18.368 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112192.log 2026-02-20T21:47:18.368 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.96217.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96217.log.gz 2026-02-20T21:47:18.368 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134046.log 2026-02-20T21:47:18.379 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112192.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57921.log 2026-02-20T21:47:18.379 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134046.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134046.log.gz 2026-02-20T21:47:18.391 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129799.log 2026-02-20T21:47:18.392 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57921.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57921.log.gz 2026-02-20T21:47:18.405 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117483.log 2026-02-20T21:47:18.405 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129799.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.129799.log.gz 2026-02-20T21:47:18.421 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140550.log 2026-02-20T21:47:18.422 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117483.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.117483.log.gz 2026-02-20T21:47:18.423 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89722.log.gz 2026-02-20T21:47:18.425 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127455.log 2026-02-20T21:47:18.426 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.140550.log: 82.5% -- replaced with /var/log/ceph/cluster1-client.mirror.140550.log.gz 2026-02-20T21:47:18.426 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115229.log 2026-02-20T21:47:18.426 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127455.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127455.log.gz 2026-02-20T21:47:18.426 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66265.log 2026-02-20T21:47:18.430 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115229.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49291.log.gz 2026-02-20T21:47:18.434 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69020.log 2026-02-20T21:47:18.434 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66265.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.66265.log.gz 2026-02-20T21:47:18.434 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58988.log 2026-02-20T21:47:18.434 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69020.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69020.log.gz 2026-02-20T21:47:18.434 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86962.log 2026-02-20T21:47:18.435 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58988.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58988.log.gz 2026-02-20T21:47:18.435 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120459.log 2026-02-20T21:47:18.435 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86962.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86962.log.gz 2026-02-20T21:47:18.435 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107329.log 2026-02-20T21:47:18.438 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120459.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120459.log.gz 2026-02-20T21:47:18.438 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53564.log 2026-02-20T21:47:18.438 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107329.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.107329.log.gz 2026-02-20T21:47:18.438 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121432.log 2026-02-20T21:47:18.438 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53564.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89699.log 2026-02-20T21:47:18.438 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53564.log.gz 2026-02-20T21:47:18.438 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121432.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.121432.log.gz 2026-02-20T21:47:18.438 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77973.log 2026-02-20T21:47:18.439 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89699.log: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89699.log.gz 2026-02-20T21:47:18.439 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42898.log 2026-02-20T21:47:18.439 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77973.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77973.log.gz 2026-02-20T21:47:18.440 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54115.log 2026-02-20T21:47:18.440 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.42898.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.42898.log.gz 2026-02-20T21:47:18.441 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99695.log 2026-02-20T21:47:18.441 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54115.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116093.log 2026-02-20T21:47:18.444 INFO:teuthology.orchestra.run.trial163.stderr: 96.1% -- replaced with /var/log/ceph/cluster2-client.mirror.54115.log.gz 2026-02-20T21:47:18.445 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99695.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99695.log.gz 2026-02-20T21:47:18.445 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88598.log 2026-02-20T21:47:18.446 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116093.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.116093.log.gz 2026-02-20T21:47:18.446 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106828.log 2026-02-20T21:47:18.450 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88598.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88598.log.gz 2026-02-20T21:47:18.451 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141123.log 2026-02-20T21:47:18.451 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106828.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106828.log.gz 2026-02-20T21:47:18.452 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65893.log 2026-02-20T21:47:18.460 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141123.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75235.log 2026-02-20T21:47:18.473 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65893.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.65893.log.gz 2026-02-20T21:47:18.480 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59729.log 2026-02-20T21:47:18.481 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75235.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75235.log.gz 2026-02-20T21:47:18.493 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136857.log 2026-02-20T21:47:18.493 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59729.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59729.log.gz 2026-02-20T21:47:18.502 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94744.log 2026-02-20T21:47:18.502 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136857.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136857.log.gz 2026-02-20T21:47:18.513 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134137.log 2026-02-20T21:47:18.513 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94744.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.112967.log.gz 2026-02-20T21:47:18.513 INFO:teuthology.orchestra.run.trial163.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.94744.log.gz 2026-02-20T21:47:18.513 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.129764.log 2026-02-20T21:47:18.515 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134137.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.134137.log.gz 2026-02-20T21:47:18.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60439.log 2026-02-20T21:47:18.516 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.129764.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.129764.log.gz 2026-02-20T21:47:18.516 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73236.log 2026-02-20T21:47:18.523 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.60439.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.60439.log.gz 2026-02-20T21:47:18.524 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47884.log 2026-02-20T21:47:18.524 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73236.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73236.log.gz 2026-02-20T21:47:18.524 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89760.log 2026-02-20T21:47:18.524 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.47884.log: 67.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47884.log.gz 2026-02-20T21:47:18.524 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77194.log 2026-02-20T21:47:18.524 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89760.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.89760.log.gz 2026-02-20T21:47:18.524 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87300.log 2026-02-20T21:47:18.524 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77194.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77194.log.gz 2026-02-20T21:47:18.524 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99645.log 2026-02-20T21:47:18.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87300.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87300.log.gz 2026-02-20T21:47:18.525 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68462.log 2026-02-20T21:47:18.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99645.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.99645.log.gz 2026-02-20T21:47:18.525 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88962.log 2026-02-20T21:47:18.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68462.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68462.log.gz 2026-02-20T21:47:18.525 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47765.log 2026-02-20T21:47:18.530 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88962.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88962.log.gz 2026-02-20T21:47:18.531 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137847.log 2026-02-20T21:47:18.531 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.47765.log: 72.8% -- replaced with /var/log/ceph/cluster1-client.admin.47765.log.gz 2026-02-20T21:47:18.531 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68497.log 2026-02-20T21:47:18.531 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137847.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137847.log.gz 2026-02-20T21:47:18.531 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113464.log 2026-02-20T21:47:18.531 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68497.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.68497.log.gz 2026-02-20T21:47:18.531 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.84203.log 2026-02-20T21:47:18.531 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113464.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113464.log.gz 2026-02-20T21:47:18.532 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138117.log 2026-02-20T21:47:18.539 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.2.84203.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70605.log 2026-02-20T21:47:18.540 INFO:teuthology.orchestra.run.trial163.stderr: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.2.84203.log.gz 2026-02-20T21:47:18.549 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78531.log 2026-02-20T21:47:18.549 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70605.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.70605.log.gz 2026-02-20T21:47:18.549 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138117.log: 86.8%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50657.log 2026-02-20T21:47:18.549 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.138117.log.gz 2026-02-20T21:47:18.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.78531.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78531.log.gz 2026-02-20T21:47:18.550 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133011.log 2026-02-20T21:47:18.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50657.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.50657.log.gz 2026-02-20T21:47:18.550 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87808.log 2026-02-20T21:47:18.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133011.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133011.log.gz 2026-02-20T21:47:18.550 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136632.log 2026-02-20T21:47:18.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87808.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.87808.log.gz 2026-02-20T21:47:18.550 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56043.log 2026-02-20T21:47:18.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136632.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.136632.log.gz 2026-02-20T21:47:18.550 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117270.log 2026-02-20T21:47:18.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56043.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56043.log.gz 2026-02-20T21:47:18.550 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138297.log 2026-02-20T21:47:18.551 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117270.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117270.log.gz 2026-02-20T21:47:18.551 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92808.log 2026-02-20T21:47:18.551 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138297.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138297.log.gz 2026-02-20T21:47:18.551 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58953.log 2026-02-20T21:47:18.551 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92808.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.92808.log.gz 2026-02-20T21:47:18.551 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94549.log 2026-02-20T21:47:18.552 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58953.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58953.log.gz 2026-02-20T21:47:18.552 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133527.log 2026-02-20T21:47:18.559 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94549.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83618.log 2026-02-20T21:47:18.568 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133527.log: 95.3% -- replaced with /var/log/ceph/cluster2-client.mirror.139433.log.gz 2026-02-20T21:47:18.570 INFO:teuthology.orchestra.run.trial163.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133527.log.gz 2026-02-20T21:47:18.570 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105491.log 2026-02-20T21:47:18.570 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83618.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83618.log.gz 2026-02-20T21:47:18.570 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123866.log 2026-02-20T21:47:18.572 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.105491.log: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.105491.log.gz 2026-02-20T21:47:18.572 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98020.log 2026-02-20T21:47:18.572 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123866.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123866.log.gz 2026-02-20T21:47:18.573 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90060.log 2026-02-20T21:47:18.575 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98020.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.98020.log.gz 2026-02-20T21:47:18.576 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131995.log 2026-02-20T21:47:18.585 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90060.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121758.log 2026-02-20T21:47:18.592 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131995.log: 88.0% 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.95801.log.gz 2026-02-20T21:47:18.593 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91128.log 2026-02-20T21:47:18.600 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121758.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.121758.log.gz 2026-02-20T21:47:18.600 INFO:teuthology.orchestra.run.trial163.stderr: 94.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.47225.log.gz 2026-02-20T21:47:18.601 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139897.log 2026-02-20T21:47:18.601 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91128.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91128.log.gz 2026-02-20T21:47:18.601 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.131995.log.gz 2026-02-20T21:47:18.601 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95412.log 2026-02-20T21:47:18.601 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139897.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68869.log 2026-02-20T21:47:18.601 INFO:teuthology.orchestra.run.trial163.stderr: 83.4% -- replaced with /var/log/ceph/cluster2-client.mirror.139897.log.gz 2026-02-20T21:47:18.601 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95412.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52748.log 2026-02-20T21:47:18.601 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.95412.log.gz 2026-02-20T21:47:18.601 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68869.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.142470.log 2026-02-20T21:47:18.601 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52748.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.68869.log.gz 2026-02-20T21:47:18.601 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52748.log.gz 2026-02-20T21:47:18.602 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47069.log 2026-02-20T21:47:18.602 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.142470.log: gzip 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.142470.log.gz 2026-02-20T21:47:18.602 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133569.log 2026-02-20T21:47:18.602 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137352.log 2026-02-20T21:47:18.602 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.47069.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47069.log.gz 2026-02-20T21:47:18.602 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133569.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103495.log 2026-02-20T21:47:18.602 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137352.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137352.log.gz 2026-02-20T21:47:18.602 INFO:teuthology.orchestra.run.trial163.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.133569.log.gz 2026-02-20T21:47:18.610 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105687.log 2026-02-20T21:47:18.610 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103495.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118430.log 2026-02-20T21:47:18.610 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.103495.log.gz 2026-02-20T21:47:18.610 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105687.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69954.log 2026-02-20T21:47:18.610 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118430.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.105687.log.gz 2026-02-20T21:47:18.610 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118430.log.gz 2026-02-20T21:47:18.610 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53857.log 2026-02-20T21:47:18.610 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69954.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63901.log 2026-02-20T21:47:18.610 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69954.log.gz 2026-02-20T21:47:18.610 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53857.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139254.log 2026-02-20T21:47:18.610 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63901.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53857.log.gz 2026-02-20T21:47:18.611 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63901.log.gz 2026-02-20T21:47:18.611 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98748.log 2026-02-20T21:47:18.611 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139254.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69299.log 2026-02-20T21:47:18.611 INFO:teuthology.orchestra.run.trial163.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.139254.log.gz 2026-02-20T21:47:18.611 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98748.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98748.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91210.log 2026-02-20T21:47:18.611 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:18.611 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54050.log 2026-02-20T21:47:18.611 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69299.log: /var/log/ceph/cluster2-client.mirror.91210.log: 84.6% 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.69299.log.gz 2026-02-20T21:47:18.611 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.91210.log.gz 2026-02-20T21:47:18.613 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76915.log 2026-02-20T21:47:18.613 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.54050.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133278.log 2026-02-20T21:47:18.613 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76915.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76915.log.gz 2026-02-20T21:47:18.613 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60084.log 2026-02-20T21:47:18.614 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133278.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.133278.log.gz 2026-02-20T21:47:18.614 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132940.log 2026-02-20T21:47:18.615 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.60084.log: 92.7% -- replaced with /var/log/ceph/cluster1-client.mirror.54050.log.gz 2026-02-20T21:47:18.616 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126269.log 2026-02-20T21:47:18.616 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132940.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.132940.log.gz 2026-02-20T21:47:18.617 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100476.log 2026-02-20T21:47:18.617 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126269.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.126269.log.gz 2026-02-20T21:47:18.618 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51182.log 2026-02-20T21:47:18.618 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100476.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.100476.log.gz 2026-02-20T21:47:18.619 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62099.log 2026-02-20T21:47:18.620 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51182.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51182.log.gz 2026-02-20T21:47:18.620 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83057.log 2026-02-20T21:47:18.620 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62099.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62099.log.gz 2026-02-20T21:47:18.622 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133995.log 2026-02-20T21:47:18.622 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83057.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.83057.log.gz 2026-02-20T21:47:18.622 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93457.log 2026-02-20T21:47:18.622 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133995.log: 97.2% -- replaced with /var/log/ceph/cluster2-client.mirror.60084.log.gz 2026-02-20T21:47:18.624 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137284.log 2026-02-20T21:47:18.624 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93457.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.133995.log.gz 2026-02-20T21:47:18.624 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130643.log 2026-02-20T21:47:18.624 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137284.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137284.log.gz 2026-02-20T21:47:18.626 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101290.log 2026-02-20T21:47:18.626 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130643.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.130643.log.gz 2026-02-20T21:47:18.626 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107213.log 2026-02-20T21:47:18.627 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101290.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101290.log.gz 2026-02-20T21:47:18.627 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.42596.log 2026-02-20T21:47:18.627 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107213.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.107213.log.gz 2026-02-20T21:47:18.629 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88227.log 2026-02-20T21:47:18.629 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.42596.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.42596.log.gz 2026-02-20T21:47:18.629 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87485.log 2026-02-20T21:47:18.632 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88227.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.88227.log.gz 2026-02-20T21:47:18.633 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100761.log 2026-02-20T21:47:18.636 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87485.log: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87485.log.gz 2026-02-20T21:47:18.637 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61448.log 2026-02-20T21:47:18.637 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100761.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.100761.log.gz 2026-02-20T21:47:18.639 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108764.log 2026-02-20T21:47:18.639 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61448.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61448.log.gz 2026-02-20T21:47:18.639 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82437.log 2026-02-20T21:47:18.639 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108764.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108764.log.gz 2026-02-20T21:47:18.641 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76056.log 2026-02-20T21:47:18.641 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82437.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82437.log.gz 2026-02-20T21:47:18.641 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133904.log 2026-02-20T21:47:18.641 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76056.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.76056.log.gz 2026-02-20T21:47:18.642 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.112192.log.gz 2026-02-20T21:47:18.642 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121800.log 2026-02-20T21:47:18.643 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133904.log: gzip -5 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.133904.log.gz 2026-02-20T21:47:18.643 INFO:teuthology.orchestra.run.trial163.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.117394.log 2026-02-20T21:47:18.649 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121800.log: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.3.47273.log.gz 2026-02-20T21:47:18.652 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66881.log 2026-02-20T21:47:18.652 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117394.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94944.log 2026-02-20T21:47:18.652 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.117394.log.gz 2026-02-20T21:47:18.652 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66881.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73692.log 2026-02-20T21:47:18.652 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94944.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66881.log.gz 2026-02-20T21:47:18.652 INFO:teuthology.orchestra.run.trial163.stderr: 1.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94944.log.gz 2026-02-20T21:47:18.652 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96659.log 2026-02-20T21:47:18.652 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.99807.log 2026-02-20T21:47:18.653 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73692.log: /var/log/ceph/cluster2-client.mirror.96659.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.96659.log.gz 2026-02-20T21:47:18.657 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95861.log 2026-02-20T21:47:18.657 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.99807.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.99807.log.gz 2026-02-20T21:47:18.657 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115628.log 2026-02-20T21:47:18.657 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95861.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95861.log.gz 2026-02-20T21:47:18.657 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103798.log 2026-02-20T21:47:18.657 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115628.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115628.log.gz 2026-02-20T21:47:18.657 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73692.log.gz 2026-02-20T21:47:18.657 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80356.log 2026-02-20T21:47:18.657 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103798.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89308.log 2026-02-20T21:47:18.657 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103798.log.gz 2026-02-20T21:47:18.657 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80356.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58080.log 2026-02-20T21:47:18.657 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.80356.log.gz 2026-02-20T21:47:18.661 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89308.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89308.log.gz 2026-02-20T21:47:18.661 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133811.log 2026-02-20T21:47:18.661 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58080.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63250.log 2026-02-20T21:47:18.661 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58080.log.gz 2026-02-20T21:47:18.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133811.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69334.log 2026-02-20T21:47:18.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63250.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63250.log.gz 2026-02-20T21:47:18.662 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.133811.log.gz 2026-02-20T21:47:18.662 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85851.log 2026-02-20T21:47:18.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69334.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65151.log 2026-02-20T21:47:18.662 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69334.log.gz 2026-02-20T21:47:18.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85851.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.45991.log 2026-02-20T21:47:18.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65151.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.85851.log.gz 2026-02-20T21:47:18.662 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77938.log 2026-02-20T21:47:18.662 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.45991.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.45991.log.gz 2026-02-20T21:47:18.662 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125741.log 2026-02-20T21:47:18.663 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77938.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77938.log.gz 2026-02-20T21:47:18.663 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128410.log 2026-02-20T21:47:18.663 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125741.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.125741.log.gz 2026-02-20T21:47:18.663 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122014.log 2026-02-20T21:47:18.664 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128410.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.128410.log.gz 2026-02-20T21:47:18.664 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65151.log.gz 2026-02-20T21:47:18.664 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.104364.log 2026-02-20T21:47:18.664 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.122014.log: -- /var/log/ceph/cluster2-client.mirror.86255.log 2026-02-20T21:47:18.664 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.122014.log.gz 2026-02-20T21:47:18.666 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.104364.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63529.log 2026-02-20T21:47:18.666 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86255.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86255.log.gz 2026-02-20T21:47:18.666 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128300.log 2026-02-20T21:47:18.666 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63529.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63529.log.gz 2026-02-20T21:47:18.667 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98534.log 2026-02-20T21:47:18.667 INFO:teuthology.orchestra.run.trial163.stderr: 95.0% -- replaced with /var/log/ceph/cluster1-client.mirror.104364.log.gz/var/log/ceph/cluster2-client.mirror.128300.log: 2026-02-20T21:47:18.667 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120630.log 2026-02-20T21:47:18.667 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98534.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98534.log.gz 2026-02-20T21:47:18.670 INFO:teuthology.orchestra.run.trial163.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128300.log.gz 2026-02-20T21:47:18.670 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121666.log 2026-02-20T21:47:18.670 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.120630.log: -- /var/log/ceph/cluster2-client.mirror.136181.log 2026-02-20T21:47:18.670 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.120630.log.gz 2026-02-20T21:47:18.670 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121666.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100395.log 2026-02-20T21:47:18.670 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136181.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121666.log.gz 2026-02-20T21:47:18.670 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97214.log 2026-02-20T21:47:18.670 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100395.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.100395.log.gz 2026-02-20T21:47:18.670 INFO:teuthology.orchestra.run.trial163.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.136181.log.gz 2026-02-20T21:47:18.670 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90418.log 2026-02-20T21:47:18.672 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66451.log 2026-02-20T21:47:18.672 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97214.log: /var/log/ceph/cluster2-client.mirror.90418.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90418.log.gz 2026-02-20T21:47:18.672 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130314.log 2026-02-20T21:47:18.672 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66451.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.66451.log.gz 2026-02-20T21:47:18.673 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89783.log 2026-02-20T21:47:18.673 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130314.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.130314.log.gz 2026-02-20T21:47:18.673 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141035.log 2026-02-20T21:47:18.673 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89783.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93722.log.gz 2026-02-20T21:47:18.674 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142514.log 2026-02-20T21:47:18.675 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141035.log: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.141035.log.gz 2026-02-20T21:47:18.676 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.129364.log 2026-02-20T21:47:18.676 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142514.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.142514.log.gz 2026-02-20T21:47:18.677 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51746.log 2026-02-20T21:47:18.678 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.129364.log: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.129364.log.gz 2026-02-20T21:47:18.678 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134095.log 2026-02-20T21:47:18.679 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51746.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51746.log.gz 2026-02-20T21:47:18.679 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94101.log 2026-02-20T21:47:18.680 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134095.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.134095.log.gz 2026-02-20T21:47:18.680 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72257.log 2026-02-20T21:47:18.691 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94101.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.125473.log 2026-02-20T21:47:18.698 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72257.log: 93.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90746.log 2026-02-20T21:47:18.698 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.72257.log.gz 2026-02-20T21:47:18.706 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.125473.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92674.log 2026-02-20T21:47:18.707 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90746.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.90746.log.gz 2026-02-20T21:47:18.707 INFO:teuthology.orchestra.run.trial163.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.125473.log.gz 2026-02-20T21:47:18.719 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113847.log 2026-02-20T21:47:18.726 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92674.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133740.log 2026-02-20T21:47:18.727 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113847.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113847.log.gz 2026-02-20T21:47:18.733 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92541.log 2026-02-20T21:47:18.734 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133740.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.133740.log.gz 2026-02-20T21:47:18.738 INFO:teuthology.orchestra.run.trial163.stderr: 95.4% -- replaced with /var/log/ceph/cluster2-client.mirror.94767.log.gz 2026-02-20T21:47:18.752 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116043.log 2026-02-20T21:47:18.753 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92541.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.92541.log.gz 2026-02-20T21:47:18.761 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75328.log 2026-02-20T21:47:18.771 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116043.log: 88.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103277.log 2026-02-20T21:47:18.774 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75328.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.116043.log.gz 2026-02-20T21:47:18.780 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133114.log 2026-02-20T21:47:18.781 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103277.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103277.log.gz 2026-02-20T21:47:18.783 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.75328.log.gz 2026-02-20T21:47:18.792 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81844.log 2026-02-20T21:47:18.793 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133114.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.133114.log.gz 2026-02-20T21:47:18.807 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110987.log 2026-02-20T21:47:18.808 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81844.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81844.log.gz 2026-02-20T21:47:18.814 INFO:teuthology.orchestra.run.trial163.stderr: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.90153.log.gz 2026-02-20T21:47:18.815 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49840.log 2026-02-20T21:47:18.816 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110987.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110987.log.gz 2026-02-20T21:47:18.816 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57242.log 2026-02-20T21:47:18.817 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49840.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49840.log.gz 2026-02-20T21:47:18.817 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48294.log 2026-02-20T21:47:18.818 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57242.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57242.log.gz 2026-02-20T21:47:18.818 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91280.log 2026-02-20T21:47:18.819 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48294.log: 62.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48294.log.gz 2026-02-20T21:47:18.819 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98418.log 2026-02-20T21:47:18.820 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91280.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91280.log.gz 2026-02-20T21:47:18.820 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62843.log 2026-02-20T21:47:18.821 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98418.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.98418.log.gz 2026-02-20T21:47:18.821 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116459.log 2026-02-20T21:47:18.822 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62843.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62843.log.gz 2026-02-20T21:47:18.822 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.141347.log 2026-02-20T21:47:18.824 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116459.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.116459.log.gz 2026-02-20T21:47:18.824 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61948.log 2026-02-20T21:47:18.824 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.141347.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.141347.log.gz 2026-02-20T21:47:18.825 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.128550.log 2026-02-20T21:47:18.827 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61948.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61948.log.gz 2026-02-20T21:47:18.827 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131582.log 2026-02-20T21:47:18.827 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128550.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.128550.log.gz 2026-02-20T21:47:18.830 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63715.log 2026-02-20T21:47:18.831 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131582.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.131582.log.gz 2026-02-20T21:47:18.831 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52376.log 2026-02-20T21:47:18.831 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63715.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63715.log.gz 2026-02-20T21:47:18.831 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71256.log 2026-02-20T21:47:18.831 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52376.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120996.log 2026-02-20T21:47:18.831 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52376.log.gz 2026-02-20T21:47:18.832 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71256.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71256.log.gz 2026-02-20T21:47:18.832 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110562.log 2026-02-20T21:47:18.841 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120996.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123049.log 2026-02-20T21:47:18.845 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110562.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.115229.log.gz 2026-02-20T21:47:18.847 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110562.log.gz 2026-02-20T21:47:18.847 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88532.log 2026-02-20T21:47:18.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123049.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123049.log.gz 2026-02-20T21:47:18.847 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91098.log.gz 2026-02-20T21:47:18.847 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127420.log 2026-02-20T21:47:18.847 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88532.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142032.log 2026-02-20T21:47:18.849 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127420.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127420.log.gz 2026-02-20T21:47:18.849 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134379.log 2026-02-20T21:47:18.853 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142032.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.142032.log.gz 2026-02-20T21:47:18.853 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.121778.log 2026-02-20T21:47:18.853 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134379.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88532.log.gz 2026-02-20T21:47:18.853 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62657.log 2026-02-20T21:47:18.853 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.121778.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.134379.log.gz 2026-02-20T21:47:18.853 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.47383.log 2026-02-20T21:47:18.853 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62657.log: 92.1% -- replaced with /var/log/ceph/cluster1-client.mirror.121778.log.gz 2026-02-20T21:47:18.853 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.62657.log.gz 2026-02-20T21:47:18.854 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86492.log 2026-02-20T21:47:18.854 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.1.47383.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.84077.log 2026-02-20T21:47:18.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86492.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.86492.log.gz 2026-02-20T21:47:18.855 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79926.log 2026-02-20T21:47:18.855 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.0.84077.log: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.1.47383.log.gz 2026-02-20T21:47:18.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140920.log 2026-02-20T21:47:18.857 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79926.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.79926.log.gz 2026-02-20T21:47:18.857 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102053.log 2026-02-20T21:47:18.859 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140920.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.140920.log.gz 2026-02-20T21:47:18.859 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72234.log 2026-02-20T21:47:18.859 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.102053.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.102053.log.gz 2026-02-20T21:47:18.859 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91693.log 2026-02-20T21:47:18.862 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.72234.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.72234.log.gz 2026-02-20T21:47:18.863 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71349.log 2026-02-20T21:47:18.876 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91693.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87417.log 2026-02-20T21:47:18.876 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71349.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71349.log.gz 2026-02-20T21:47:18.885 INFO:teuthology.orchestra.run.trial163.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.91693.log.gz 2026-02-20T21:47:18.885 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87265.log 2026-02-20T21:47:18.886 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.87417.log: 88.7% -- replaced with /var/log/ceph/cluster1-client.mirror.87417.log.gz 2026-02-20T21:47:18.886 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77287.log 2026-02-20T21:47:18.890 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.87265.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87265.log.gz 2026-02-20T21:47:18.890 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115392.log 2026-02-20T21:47:18.890 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77287.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77287.log.gz 2026-02-20T21:47:18.890 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.125861.log 2026-02-20T21:47:18.890 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115392.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115392.log.gz 2026-02-20T21:47:18.891 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115663.log 2026-02-20T21:47:18.891 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.125861.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.125861.log.gz 2026-02-20T21:47:18.891 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57789.log 2026-02-20T21:47:18.891 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115663.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.115663.log.gz 2026-02-20T21:47:18.891 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64796.log 2026-02-20T21:47:18.893 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57789.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.57789.log.gz 2026-02-20T21:47:18.893 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136106.log 2026-02-20T21:47:18.893 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64796.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64796.log.gz 2026-02-20T21:47:18.893 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.126504.log 2026-02-20T21:47:18.895 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136106.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.136106.log.gz 2026-02-20T21:47:18.895 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.111375.log 2026-02-20T21:47:18.897 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.126504.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.126504.log.gz 2026-02-20T21:47:18.897 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57401.log 2026-02-20T21:47:18.905 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.0.111375.log: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.91303.log.gz 2026-02-20T21:47:18.908 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88095.log 2026-02-20T21:47:18.908 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.57401.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.57401.log.gz 2026-02-20T21:47:18.908 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.113094.log 2026-02-20T21:47:18.912 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88095.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.mirror.88095.log.gz 2026-02-20T21:47:18.912 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90829.log 2026-02-20T21:47:18.912 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.113094.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.113094.log.gz 2026-02-20T21:47:18.912 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47799.log 2026-02-20T21:47:18.913 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90829.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.90829.log.gz 2026-02-20T21:47:18.913 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88121.log 2026-02-20T21:47:18.913 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.47799.log: 5.8% -- replaced with /var/log/ceph/cluster1-client.mirror.47799.log.gz 2026-02-20T21:47:18.913 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95228.log 2026-02-20T21:47:18.919 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88121.log: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88121.log.gz 2026-02-20T21:47:18.919 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137802.log 2026-02-20T21:47:18.919 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.95228.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.95228.log.gz 2026-02-20T21:47:18.919 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47707.log 2026-02-20T21:47:18.919 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137802.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.137802.log.gz 2026-02-20T21:47:18.920 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79426.log 2026-02-20T21:47:18.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.47707.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47707.log.gz 2026-02-20T21:47:18.920 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104760.log 2026-02-20T21:47:18.920 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79426.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79426.log.gz 2026-02-20T21:47:18.920 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52936.log 2026-02-20T21:47:18.922 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104760.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.104760.log.gz 2026-02-20T21:47:18.922 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117783.log 2026-02-20T21:47:18.922 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52936.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110177.log 2026-02-20T21:47:18.923 INFO:teuthology.orchestra.run.trial163.stderr: 95.4% -- replaced with /var/log/ceph/cluster2-client.mirror.52936.log.gz 2026-02-20T21:47:18.926 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117783.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.117783.log.gz 2026-02-20T21:47:18.926 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116128.log 2026-02-20T21:47:18.926 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110177.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110177.log.gz 2026-02-20T21:47:18.926 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.112125.log 2026-02-20T21:47:18.927 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116128.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.116128.log.gz 2026-02-20T21:47:18.927 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75421.log 2026-02-20T21:47:18.929 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.112125.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.112125.log.gz 2026-02-20T21:47:18.929 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113710.log 2026-02-20T21:47:18.929 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75421.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75421.log.gz 2026-02-20T21:47:18.929 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102376.log 2026-02-20T21:47:18.931 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113710.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113710.log.gz 2026-02-20T21:47:18.931 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104069.log 2026-02-20T21:47:18.932 INFO:teuthology.orchestra.run.trial163.stderr: 93.3% -- replaced with /var/log/ceph/cluster1-client.mirror.0.111375.log.gz 2026-02-20T21:47:18.932 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102376.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94146.log 2026-02-20T21:47:18.932 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104069.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104069.log.gz 2026-02-20T21:47:18.932 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59942.log 2026-02-20T21:47:18.932 INFO:teuthology.orchestra.run.trial163.stderr: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.102376.log.gz 2026-02-20T21:47:18.932 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.94146.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.65558.log 2026-02-20T21:47:18.933 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59942.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59942.log.gz 2026-02-20T21:47:18.933 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133833.log 2026-02-20T21:47:18.935 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.1.65558.log: 93.4% -- replaced with /var/log/ceph/cluster1-client.mirror.1.65558.log.gz 2026-02-20T21:47:18.935 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91075.log 2026-02-20T21:47:18.937 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133833.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133833.log.gz 2026-02-20T21:47:18.937 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76369.log 2026-02-20T21:47:18.937 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91075.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91075.log.gz 2026-02-20T21:47:18.938 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91374.log 2026-02-20T21:47:18.938 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76369.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76369.log.gz 2026-02-20T21:47:18.938 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74259.log 2026-02-20T21:47:18.942 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.91374.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.94146.log.gz 2026-02-20T21:47:18.943 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102098.log 2026-02-20T21:47:18.944 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74259.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74259.log.gz 2026-02-20T21:47:18.944 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102804.log 2026-02-20T21:47:18.947 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.102098.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.102098.log.gz 2026-02-20T21:47:18.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70419.log 2026-02-20T21:47:18.947 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102804.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.102804.log.gz 2026-02-20T21:47:18.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119243.log 2026-02-20T21:47:18.947 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70419.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70419.log.gz 2026-02-20T21:47:18.947 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74564.log 2026-02-20T21:47:18.947 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119243.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.119243.log.gz 2026-02-20T21:47:18.949 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109928.log 2026-02-20T21:47:18.949 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.74564.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.74564.log.gz 2026-02-20T21:47:18.949 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79089.log 2026-02-20T21:47:18.957 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109928.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72122.log 2026-02-20T21:47:18.958 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79089.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79089.log.gz 2026-02-20T21:47:18.961 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.141123.log.gz 2026-02-20T21:47:18.964 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75851.log 2026-02-20T21:47:18.964 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72122.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.72122.log.gz 2026-02-20T21:47:18.964 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103176.log 2026-02-20T21:47:18.967 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75851.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75851.log.gz 2026-02-20T21:47:18.967 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48452.log 2026-02-20T21:47:18.967 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103176.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103176.log.gz 2026-02-20T21:47:18.967 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.76615.log 2026-02-20T21:47:18.967 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48452.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58371.log 2026-02-20T21:47:18.967 INFO:teuthology.orchestra.run.trial163.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.48452.log.gz 2026-02-20T21:47:18.974 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.1.76615.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137712.log 2026-02-20T21:47:18.976 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58371.log: 88.3% 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.139229.log.gz 2026-02-20T21:47:18.976 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.58371.log.gz 2026-02-20T21:47:18.979 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69427.log 2026-02-20T21:47:18.979 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137712.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137712.log.gz 2026-02-20T21:47:18.979 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.109906.log 2026-02-20T21:47:18.979 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69427.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69427.log.gz 2026-02-20T21:47:18.979 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93993.log 2026-02-20T21:47:18.980 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.109906.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.109906.log.gz 2026-02-20T21:47:18.980 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131817.log 2026-02-20T21:47:19.002 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93993.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107921.log 2026-02-20T21:47:19.002 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131817.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131817.log.gz 2026-02-20T21:47:19.015 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77508.log 2026-02-20T21:47:19.015 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107921.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.107921.log.gz 2026-02-20T21:47:19.022 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76569.log 2026-02-20T21:47:19.023 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77508.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77508.log.gz 2026-02-20T21:47:19.026 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.109928.log.gz 2026-02-20T21:47:19.026 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53021.log 2026-02-20T21:47:19.035 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.76569.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134066.log 2026-02-20T21:47:19.047 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.53021.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.53021.log.gz 2026-02-20T21:47:19.050 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118147.log 2026-02-20T21:47:19.052 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134066.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.134066.log.gz 2026-02-20T21:47:19.064 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70268.log 2026-02-20T21:47:19.065 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.118147.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.118147.log.gz 2026-02-20T21:47:19.066 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90567.log.gz 2026-02-20T21:47:19.081 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138522.log 2026-02-20T21:47:19.081 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70268.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70268.log.gz 2026-02-20T21:47:19.081 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52341.log 2026-02-20T21:47:19.082 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138522.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138522.log.gz 2026-02-20T21:47:19.082 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95311.log 2026-02-20T21:47:19.082 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52341.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.52341.log.gz 2026-02-20T21:47:19.082 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64889.log 2026-02-20T21:47:19.082 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95311.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95311.log.gz 2026-02-20T21:47:19.082 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59244.log 2026-02-20T21:47:19.082 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64889.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64889.log.gz 2026-02-20T21:47:19.082 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49945.log 2026-02-20T21:47:19.082 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59244.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.59244.log.gz 2026-02-20T21:47:19.082 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50482.log 2026-02-20T21:47:19.082 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49945.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137509.log 2026-02-20T21:47:19.082 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49945.log.gz 2026-02-20T21:47:19.083 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50482.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56175.log 2026-02-20T21:47:19.083 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50482.log.gz 2026-02-20T21:47:19.083 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137509.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.137509.log.gz 2026-02-20T21:47:19.083 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46387.log 2026-02-20T21:47:19.083 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56175.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56175.log.gz 2026-02-20T21:47:19.083 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130589.log 2026-02-20T21:47:19.083 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.46387.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46387.log.gz 2026-02-20T21:47:19.083 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70326.log 2026-02-20T21:47:19.083 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130589.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130589.log.gz 2026-02-20T21:47:19.083 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.119930.log 2026-02-20T21:47:19.083 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70326.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70326.log.gz 2026-02-20T21:47:19.083 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56563.log 2026-02-20T21:47:19.084 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.119930.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.119930.log.gz 2026-02-20T21:47:19.084 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133669.log 2026-02-20T21:47:19.084 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56563.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56563.log.gz 2026-02-20T21:47:19.084 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104975.log 2026-02-20T21:47:19.087 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133669.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.133669.log.gz 2026-02-20T21:47:19.087 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66730.log 2026-02-20T21:47:19.087 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104975.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104975.log.gz 2026-02-20T21:47:19.087 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88747.log 2026-02-20T21:47:19.087 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66730.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.66730.log.gz 2026-02-20T21:47:19.087 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51851.log 2026-02-20T21:47:19.090 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88747.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88747.log.gz 2026-02-20T21:47:19.090 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47423.log 2026-02-20T21:47:19.090 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51851.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138679.log 2026-02-20T21:47:19.090 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.51851.log.gz 2026-02-20T21:47:19.093 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.47423.log: 94.7% -- replaced with /var/log/ceph/cluster1-client.mirror.47423.log.gz 2026-02-20T21:47:19.094 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71926.log 2026-02-20T21:47:19.094 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138679.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138679.log.gz 2026-02-20T21:47:19.094 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138184.log 2026-02-20T21:47:19.096 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71926.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.71926.log.gz 2026-02-20T21:47:19.096 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103678.log 2026-02-20T21:47:19.097 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138184.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.138184.log.gz 2026-02-20T21:47:19.097 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139203.log 2026-02-20T21:47:19.097 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.103678.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103678.log.gz 2026-02-20T21:47:19.097 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136015.log 2026-02-20T21:47:19.100 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139203.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.139203.log.gz 2026-02-20T21:47:19.100 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97375.log 2026-02-20T21:47:19.100 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136015.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.136015.log.gz 2026-02-20T21:47:19.101 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54619.log 2026-02-20T21:47:19.101 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97375.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97375.log.gz 2026-02-20T21:47:19.101 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88661.log 2026-02-20T21:47:19.101 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54619.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.54619.log.gz 2026-02-20T21:47:19.101 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56528.log 2026-02-20T21:47:19.104 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88661.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88661.log.gz 2026-02-20T21:47:19.104 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139456.log 2026-02-20T21:47:19.106 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56528.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56528.log.gz 2026-02-20T21:47:19.106 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134614.log 2026-02-20T21:47:19.106 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139456.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.139456.log.gz 2026-02-20T21:47:19.107 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62320.log 2026-02-20T21:47:19.109 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134614.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134614.log.gz 2026-02-20T21:47:19.109 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.102396.log 2026-02-20T21:47:19.109 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.62320.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.62320.log.gz 2026-02-20T21:47:19.109 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90238.log 2026-02-20T21:47:19.109 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.102396.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.120996.log.gz 2026-02-20T21:47:19.112 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86558.log 2026-02-20T21:47:19.112 INFO:teuthology.orchestra.run.trial163.stderr: 92.5%/var/log/ceph/cluster2-client.mirror.90238.log: -- replaced with /var/log/ceph/cluster1-client.mirror.102396.log.gz 2026-02-20T21:47:19.112 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80019.log 2026-02-20T21:47:19.112 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86558.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86558.log.gz 2026-02-20T21:47:19.112 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88270.log 2026-02-20T21:47:19.112 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80019.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80019.log.gz 2026-02-20T21:47:19.113 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69241.log 2026-02-20T21:47:19.117 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88270.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.88270.log.gz 2026-02-20T21:47:19.117 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52271.log 2026-02-20T21:47:19.117 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69241.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69241.log.gz 2026-02-20T21:47:19.117 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93160.log 2026-02-20T21:47:19.120 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52271.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52271.log.gz 2026-02-20T21:47:19.120 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89132.log 2026-02-20T21:47:19.120 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93160.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.93160.log.gz 2026-02-20T21:47:19.120 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60019.log 2026-02-20T21:47:19.120 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.89132.log: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.89132.log.gz 2026-02-20T21:47:19.120 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80391.log 2026-02-20T21:47:19.123 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.60019.log: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.60019.log.gz 2026-02-20T21:47:19.123 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.105188.log 2026-02-20T21:47:19.124 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80391.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.80391.log.gz 2026-02-20T21:47:19.124 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.136232.log 2026-02-20T21:47:19.127 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.105188.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.105188.log.gz 2026-02-20T21:47:19.127 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46579.log 2026-02-20T21:47:19.127 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.136232.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.136232.log.gz 2026-02-20T21:47:19.127 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79554.log 2026-02-20T21:47:19.127 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.46579.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46579.log.gz 2026-02-20T21:47:19.127 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138612.log 2026-02-20T21:47:19.127 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.79554.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79554.log.gz 2026-02-20T21:47:19.130 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110775.log 2026-02-20T21:47:19.130 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138612.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.138612.log.gz 2026-02-20T21:47:19.130 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.121972.log 2026-02-20T21:47:19.130 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110775.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110775.log.gz 2026-02-20T21:47:19.130 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.94549.log.gz 2026-02-20T21:47:19.130 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70512.log 2026-02-20T21:47:19.130 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.121972.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104554.log 2026-02-20T21:47:19.130 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.121972.log.gz 2026-02-20T21:47:19.131 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70512.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.70512.log.gz 2026-02-20T21:47:19.133 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.103410.log 2026-02-20T21:47:19.133 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136924.log 2026-02-20T21:47:19.133 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104554.log: /var/log/ceph/cluster2-client.mirror.103410.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104554.log.gz 2026-02-20T21:47:19.133 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.103410.log.gz 2026-02-20T21:47:19.133 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48277.log 2026-02-20T21:47:19.133 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140868.log 2026-02-20T21:47:19.133 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136924.log: /var/log/ceph/cluster2-client.admin.48277.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.136924.log.gz 2026-02-20T21:47:19.133 INFO:teuthology.orchestra.run.trial163.stderr: 72.9% -- replaced with /var/log/ceph/cluster2-client.admin.48277.log.gz 2026-02-20T21:47:19.133 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134521.log 2026-02-20T21:47:19.134 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73015.log 2026-02-20T21:47:19.134 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.140868.log: /var/log/ceph/cluster2-client.mirror.134521.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.140868.log.gz 2026-02-20T21:47:19.137 INFO:teuthology.orchestra.run.trial163.stderr:gzip 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.134521.log.gz 2026-02-20T21:47:19.137 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120424.log 2026-02-20T21:47:19.137 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73015.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.73015.log.gz 2026-02-20T21:47:19.137 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92488.log 2026-02-20T21:47:19.137 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61483.log 2026-02-20T21:47:19.137 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120424.log: /var/log/ceph/cluster2-client.mirror.92488.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120424.log.gz 2026-02-20T21:47:19.137 INFO:teuthology.orchestra.run.trial163.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.92488.log.gz 2026-02-20T21:47:19.137 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.100818.log 2026-02-20T21:47:19.137 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61483.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.61483.log.gz 2026-02-20T21:47:19.138 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65451.log 2026-02-20T21:47:19.138 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.100818.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55205.log 2026-02-20T21:47:19.138 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.65451.log: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.65451.log.gz 2026-02-20T21:47:19.139 INFO:teuthology.orchestra.run.trial163.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.100818.log.gz 2026-02-20T21:47:19.139 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54005.log 2026-02-20T21:47:19.139 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.55205.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55205.log.gz 2026-02-20T21:47:19.140 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.137667.log 2026-02-20T21:47:19.140 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.54005.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67474.log 2026-02-20T21:47:19.140 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.54005.log.gz 2026-02-20T21:47:19.140 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.137667.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.137667.log.gz 2026-02-20T21:47:19.141 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136812.log 2026-02-20T21:47:19.141 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67474.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67474.log.gz 2026-02-20T21:47:19.141 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140641.log 2026-02-20T21:47:19.141 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136812.log: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.136812.log.gz 2026-02-20T21:47:19.141 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.118542.log 2026-02-20T21:47:19.142 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.140641.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.140641.log.gz 2026-02-20T21:47:19.142 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92129.log 2026-02-20T21:47:19.143 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.118542.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.118542.log.gz 2026-02-20T21:47:19.143 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133853.log 2026-02-20T21:47:19.143 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92129.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.92129.log.gz 2026-02-20T21:47:19.144 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59999.log 2026-02-20T21:47:19.145 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133853.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.133853.log.gz 2026-02-20T21:47:19.146 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48362.log 2026-02-20T21:47:19.149 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59999.log: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.59999.log.gz 2026-02-20T21:47:19.150 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83914.log 2026-02-20T21:47:19.150 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.48362.log: 67.9% -- replaced with /var/log/ceph/cluster2-client.mirror.48362.log.gz 2026-02-20T21:47:19.151 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52026.log 2026-02-20T21:47:19.151 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142658.log 2026-02-20T21:47:19.152 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.83914.log: /var/log/ceph/cluster2-client.mirror.52026.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83970.log 2026-02-20T21:47:19.152 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52026.log.gz 2026-02-20T21:47:19.152 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142658.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.142658.log.gz 2026-02-20T21:47:19.153 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131247.log 2026-02-20T21:47:19.153 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.83970.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.admin.83970.log.gz 2026-02-20T21:47:19.154 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88704.log 2026-02-20T21:47:19.155 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131247.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.131247.log.gz 2026-02-20T21:47:19.155 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose 96.5% -- replaced with /var/log/ceph/cluster2-client.mirror.83914.log.gz 2026-02-20T21:47:19.155 INFO:teuthology.orchestra.run.trial163.stderr: -- /var/log/ceph/cluster1-client.mirror.95734.log 2026-02-20T21:47:19.155 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88704.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56334.log 2026-02-20T21:47:19.156 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.95734.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.95734.log.gz 2026-02-20T21:47:19.156 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135854.log 2026-02-20T21:47:19.157 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56334.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.56334.log.gz 2026-02-20T21:47:19.157 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97580.log 2026-02-20T21:47:19.158 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135854.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.135854.log.gz 2026-02-20T21:47:19.158 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88704.log.gz 2026-02-20T21:47:19.158 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58212.log 2026-02-20T21:47:19.159 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97580.log: gzip 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.97580.log.gz 2026-02-20T21:47:19.159 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47833.log 2026-02-20T21:47:19.161 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58212.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58212.log.gz 2026-02-20T21:47:19.161 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95896.log 2026-02-20T21:47:19.161 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.47833.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132818.log 2026-02-20T21:47:19.161 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.95896.log: 5.8% -- replaced with /var/log/ceph/cluster1-client.mirror.47833.log.gz 2026-02-20T21:47:19.162 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140994.log 2026-02-20T21:47:19.162 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.95896.log.gz 2026-02-20T21:47:19.162 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132818.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53811.log 2026-02-20T21:47:19.162 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140994.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.140994.log.gz 2026-02-20T21:47:19.164 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113913.log 2026-02-20T21:47:19.165 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.53811.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.53811.log.gz 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.132818.log.gz 2026-02-20T21:47:19.165 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:19.165 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.109506.log 2026-02-20T21:47:19.165 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113913.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123789.log 2026-02-20T21:47:19.165 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.113913.log.gz 2026-02-20T21:47:19.165 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.109506.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85482.log 2026-02-20T21:47:19.165 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123789.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.109506.log.gz 2026-02-20T21:47:19.165 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123789.log.gz 2026-02-20T21:47:19.165 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.104364.log 2026-02-20T21:47:19.165 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113422.log 2026-02-20T21:47:19.165 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85482.log: /var/log/ceph/cluster2-client.mirror.0.104364.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.85482.log.gz 2026-02-20T21:47:19.166 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.47782.log 2026-02-20T21:47:19.166 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113422.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113422.log.gz 2026-02-20T21:47:19.166 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99475.log 2026-02-20T21:47:19.167 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.47782.log: 62.5% -- replaced with /var/log/ceph/cluster1-client.mirror.47782.log.gz 2026-02-20T21:47:19.167 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130146.log 2026-02-20T21:47:19.168 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99475.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99475.log.gz 2026-02-20T21:47:19.168 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63157.log 2026-02-20T21:47:19.170 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130146.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130146.log.gz 2026-02-20T21:47:19.170 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.116501.log 2026-02-20T21:47:19.170 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63157.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63157.log.gz 2026-02-20T21:47:19.170 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.118628.log 2026-02-20T21:47:19.178 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.116501.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132457.log 2026-02-20T21:47:19.190 INFO:teuthology.orchestra.run.trial163.stderr: 94.1% -- replaced with /var/log/ceph/cluster2-client.mirror.0.104364.log.gz 2026-02-20T21:47:19.191 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65094.log 2026-02-20T21:47:19.192 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132457.log: /var/log/ceph/cluster2-client.mirror.118628.log: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.132457.log.gz 2026-02-20T21:47:19.193 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.111931.log 2026-02-20T21:47:19.193 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.121800.log.gz 2026-02-20T21:47:19.193 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.65094.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65094.log.gz 2026-02-20T21:47:19.194 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74093.log 2026-02-20T21:47:19.194 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.111931.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.111931.log.gz 2026-02-20T21:47:19.194 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82627.log 2026-02-20T21:47:19.195 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74093.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74093.log.gz 2026-02-20T21:47:19.195 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90937.log 2026-02-20T21:47:19.196 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82627.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82627.log.gz 2026-02-20T21:47:19.196 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140251.log 2026-02-20T21:47:19.205 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90937.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.122441.log 2026-02-20T21:47:19.206 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140251.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.140251.log.gz 2026-02-20T21:47:19.218 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59974.log 2026-02-20T21:47:19.219 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.122441.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.122441.log.gz 2026-02-20T21:47:19.235 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130359.log 2026-02-20T21:47:19.235 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.59974.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.59974.log.gz 2026-02-20T21:47:19.244 INFO:teuthology.orchestra.run.trial163.stderr: 95.9% -- replaced with /var/log/ceph/cluster2-client.mirror.83937.log.gz 2026-02-20T21:47:19.244 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72003.log 2026-02-20T21:47:19.245 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130359.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.130359.log.gz 2026-02-20T21:47:19.246 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50587.log 2026-02-20T21:47:19.246 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72003.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.72003.log.gz 2026-02-20T21:47:19.247 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.131729.log 2026-02-20T21:47:19.247 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50587.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50587.log.gz 2026-02-20T21:47:19.247 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99730.log 2026-02-20T21:47:19.256 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.131729.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84433.log 2026-02-20T21:47:19.256 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99730.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99730.log.gz 2026-02-20T21:47:19.265 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86088.log 2026-02-20T21:47:19.276 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84433.log: 89.1%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.120208.log 2026-02-20T21:47:19.276 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86088.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86088.log.gz 2026-02-20T21:47:19.278 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.84433.log.gz 2026-02-20T21:47:19.282 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.97214.log.gz 2026-02-20T21:47:19.286 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107685.log 2026-02-20T21:47:19.286 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.120208.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.120208.log.gz 2026-02-20T21:47:19.286 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67846.log 2026-02-20T21:47:19.286 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107685.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107685.log.gz 2026-02-20T21:47:19.286 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.99874.log 2026-02-20T21:47:19.286 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67846.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67846.log.gz 2026-02-20T21:47:19.286 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.139724.log 2026-02-20T21:47:19.301 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.1.99874.log: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.94101.log.gz 2026-02-20T21:47:19.301 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133924.log 2026-02-20T21:47:19.301 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.139724.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.139724.log.gz 2026-02-20T21:47:19.302 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.106420.log 2026-02-20T21:47:19.303 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133924.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.133924.log.gz 2026-02-20T21:47:19.303 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113062.log 2026-02-20T21:47:19.314 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.106420.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.106420.log.gz 2026-02-20T21:47:19.314 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63866.log 2026-02-20T21:47:19.314 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113062.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.113062.log.gz 2026-02-20T21:47:19.314 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142555.log 2026-02-20T21:47:19.314 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63866.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.63866.log.gz 2026-02-20T21:47:19.314 INFO:teuthology.orchestra.run.trial163.stderr: 95.5%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.117153.log 2026-02-20T21:47:19.314 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.91374.log.gz 2026-02-20T21:47:19.314 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85346.log 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.117153.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.117153.log.gz 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85043.log 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85346.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85346.log.gz 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.142555.log.gz 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112868.log 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85043.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134947.log 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.85043.log.gz 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112868.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58503.log 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134947.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112868.log.gz 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr:gzip 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.134947.log.gz 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61018.log 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92511.log 2026-02-20T21:47:19.315 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58503.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58503.log.gz 2026-02-20T21:47:19.316 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61018.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61018.log.gz 2026-02-20T21:47:19.316 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135569.log 2026-02-20T21:47:19.320 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.92511.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124479.log 2026-02-20T21:47:19.325 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135569.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.135569.log.gz 2026-02-20T21:47:19.325 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.110740.log 2026-02-20T21:47:19.325 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124479.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124479.log.gz 2026-02-20T21:47:19.325 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72546.log 2026-02-20T21:47:19.325 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.110740.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.110740.log.gz 2026-02-20T21:47:19.325 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102120.log 2026-02-20T21:47:19.325 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.72546.log: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.72546.log.gz 2026-02-20T21:47:19.325 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.142326.log 2026-02-20T21:47:19.335 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.102120.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61727.log 2026-02-20T21:47:19.336 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.142326.log: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.142326.log.gz 2026-02-20T21:47:19.340 INFO:teuthology.orchestra.run.trial163.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-client.mirror.1.99874.log.gz 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.92674.log.gz 2026-02-20T21:47:19.340 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:19.341 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61820.log 2026-02-20T21:47:19.346 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61727.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80821.log 2026-02-20T21:47:19.346 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61727.log.gz 2026-02-20T21:47:19.346 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61820.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138702.log 2026-02-20T21:47:19.346 INFO:teuthology.orchestra.run.trial163.stderr: 88.6%/var/log/ceph/cluster2-client.mirror.80821.log: -- replaced with /var/log/ceph/cluster2-client.mirror.61820.log.gz 2026-02-20T21:47:19.347 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80821.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49735.log 2026-02-20T21:47:19.347 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:19.347 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77101.log 2026-02-20T21:47:19.347 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138702.log: /var/log/ceph/cluster2-client.mirror.49735.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138702.log.gz 2026-02-20T21:47:19.347 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.45754.log 2026-02-20T21:47:19.347 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77101.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49735.log.gz 2026-02-20T21:47:19.347 INFO:teuthology.orchestra.run.trial163.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77101.log.gz 2026-02-20T21:47:19.347 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132122.log 2026-02-20T21:47:19.347 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.45754.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.45754.log.gz 2026-02-20T21:47:19.348 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73888.log 2026-02-20T21:47:19.348 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123670.log 2026-02-20T21:47:19.348 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132122.log: /var/log/ceph/cluster2-client.mirror.73888.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.132122.log.gz 2026-02-20T21:47:19.348 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.73888.log.gz 2026-02-20T21:47:19.376 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54425.log 2026-02-20T21:47:19.376 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123670.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75665.log 2026-02-20T21:47:19.376 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123670.log.gz 2026-02-20T21:47:19.376 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.54425.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141056.log 2026-02-20T21:47:19.376 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.54425.log.gz 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75665.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75665.log.gz 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90215.log 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141056.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75793.log 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr: 83.4% -- replaced with /var/log/ceph/cluster2-client.mirror.141056.log.gz 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.90215.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50120.log 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75793.log: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.90215.log.gz 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75793.log.gz 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53351.log 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.50120.log: /var/log/ceph/cluster2-client.mirror.128515.log 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59570.log 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.128515.log: /var/log/ceph/cluster2-client.mirror.53351.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.50120.log.gz 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53351.log.gz 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.128515.log.gz 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58856.log 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138162.log 2026-02-20T21:47:19.377 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59570.log: /var/log/ceph/cluster2-client.mirror.58856.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59570.log.gz 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.58856.log.gz 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46824.log 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.105446.log 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138162.log: /var/log/ceph/cluster1-client.admin.46824.log: 86.8% 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46824.log.gz 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster1-client.mirror.138162.log.gz 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.97410.log 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.105446.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71105.log 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.105446.log.gz 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.97410.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.97410.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133640.log 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71105.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140210.log 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.71105.log.gz 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104465.log 2026-02-20T21:47:19.378 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140210.log: 83.3% -- replaced with /var/log/ceph/cluster2-client.mirror.140210.log.gz 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99204.log 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104465.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.104465.log.gz 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr: 90.6%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.52131.log 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.133640.log.gz 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99204.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71442.log 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.52131.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.99204.log.gz 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69055.log 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.52131.log.gz 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71721.log 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71442.log: 88.6%/var/log/ceph/cluster2-client.mirror.69055.log: -- replaced with /var/log/ceph/cluster2-client.mirror.71442.log.gz 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69055.log.gz 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.138780.log 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73515.log 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.71721.log: /var/log/ceph/cluster2-client.mirror.138780.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71721.log.gz 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.138780.log.gz 2026-02-20T21:47:19.379 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134401.log 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113653.log 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.73515.log: /var/log/ceph/cluster2-client.mirror.134401.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.73515.log.gz 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.134401.log.gz 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.138634.log 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113653.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67567.log 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113653.log.gz 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.138634.log: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.138634.log.gz 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78903.log 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86154.log 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67567.log: /var/log/ceph/cluster2-client.mirror.78903.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.67567.log.gz 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.78903.log.gz 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87587.log 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53776.log 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86154.log: /var/log/ceph/cluster2-client.mirror.87587.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.86154.log.gz 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87587.log.gz 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75514.log 2026-02-20T21:47:19.380 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.mirror.53776.log: /var/log/ceph/cluster2-client.mirror.124000.log 2026-02-20T21:47:19.381 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75514.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53776.log.gz 2026-02-20T21:47:19.381 INFO:teuthology.orchestra.run.trial163.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75514.log.gzgzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.140814.log 2026-02-20T21:47:19.381 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:47:19.383 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124000.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69613.log 2026-02-20T21:47:19.383 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.140814.log: 97.3% -- replaced with /var/log/ceph/cluster2-client.mirror.124000.log.gz 2026-02-20T21:47:19.383 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51077.log 2026-02-20T21:47:19.383 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69613.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69613.log.gz 2026-02-20T21:47:19.383 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.135151.log 2026-02-20T21:47:19.383 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51077.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.112448.log 2026-02-20T21:47:19.383 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51077.log.gz 2026-02-20T21:47:19.384 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.135151.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.135151.log.gz 2026-02-20T21:47:19.384 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124104.log 2026-02-20T21:47:19.384 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.112448.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.112448.log.gz 2026-02-20T21:47:19.386 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84367.log 2026-02-20T21:47:19.386 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124104.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124104.log.gz 2026-02-20T21:47:19.386 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51921.log 2026-02-20T21:47:19.387 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.84367.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.84367.log.gz 2026-02-20T21:47:19.387 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.124992.log 2026-02-20T21:47:19.387 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.51921.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56916.log 2026-02-20T21:47:19.387 INFO:teuthology.orchestra.run.trial163.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.51921.log.gz 2026-02-20T21:47:19.388 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.124992.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.124992.log.gz 2026-02-20T21:47:19.388 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.104870.log 2026-02-20T21:47:19.388 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56916.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56916.log.gz 2026-02-20T21:47:19.390 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132889.log 2026-02-20T21:47:19.390 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.104870.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.104870.log.gz 2026-02-20T21:47:19.390 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91426.log 2026-02-20T21:47:19.391 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132889.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.132889.log.gz 2026-02-20T21:47:19.392 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74336.log 2026-02-20T21:47:19.392 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.91426.log: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.91426.log.gz 2026-02-20T21:47:19.392 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85785.log 2026-02-20T21:47:19.393 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74336.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.74336.log.gz 2026-02-20T21:47:19.393 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59764.log 2026-02-20T21:47:19.394 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85785.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.85785.log.gz 2026-02-20T21:47:19.394 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.92395.log 2026-02-20T21:47:19.395 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59764.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59764.log.gz 2026-02-20T21:47:19.395 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82964.log 2026-02-20T21:47:19.404 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.2.92395.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.113576.log 2026-02-20T21:47:19.418 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.82964.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82964.log.gz 2026-02-20T21:47:19.420 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49805.log 2026-02-20T21:47:19.421 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.113576.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.113576.log.gz 2026-02-20T21:47:19.427 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115121.log 2026-02-20T21:47:19.440 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.49805.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49805.log.gz 2026-02-20T21:47:19.441 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130554.log 2026-02-20T21:47:19.441 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115121.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.115121.log.gz 2026-02-20T21:47:19.457 INFO:teuthology.orchestra.run.trial163.stderr: 94.2%gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85684.log 2026-02-20T21:47:19.457 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130554.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130554.log.gz 2026-02-20T21:47:19.457 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.2.92395.log.gz 2026-02-20T21:47:19.457 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.124637.log 2026-02-20T21:47:19.457 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85684.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85684.log.gz 2026-02-20T21:47:19.457 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88207.log 2026-02-20T21:47:19.458 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.124637.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.124637.log.gz 2026-02-20T21:47:19.459 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122854.log 2026-02-20T21:47:19.466 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88207.log: 93.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88207.log.gz 2026-02-20T21:47:19.466 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75200.log 2026-02-20T21:47:19.466 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122854.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.122854.log.gz 2026-02-20T21:47:19.466 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106882.log 2026-02-20T21:47:19.466 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75200.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75200.log.gz 2026-02-20T21:47:19.466 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61576.log 2026-02-20T21:47:19.467 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106882.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.106882.log.gz 2026-02-20T21:47:19.467 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63680.log 2026-02-20T21:47:19.467 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61576.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61576.log.gz 2026-02-20T21:47:19.480 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.130888.log 2026-02-20T21:47:19.480 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.63680.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.63680.log.gz 2026-02-20T21:47:19.480 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127786.log 2026-02-20T21:47:19.480 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.130888.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.130888.log.gz 2026-02-20T21:47:19.481 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132400.log 2026-02-20T21:47:19.481 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127786.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127786.log.gz 2026-02-20T21:47:19.481 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127032.log 2026-02-20T21:47:19.481 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132400.log: 95.5% 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.132400.log.gz 2026-02-20T21:47:19.481 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.131729.log.gz 2026-02-20T21:47:19.481 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.143410.log 2026-02-20T21:47:19.481 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115040.log 2026-02-20T21:47:19.481 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.127032.log: /var/log/ceph/cluster2-client.admin.143410.log: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.127032.log.gz 2026-02-20T21:47:19.481 INFO:teuthology.orchestra.run.trial163.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.143410.log.gz 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.106510.log 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115040.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61855.log 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.115040.log.gz 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.106510.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132645.log 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.61855.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.106510.log.gz 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.61855.log.gz 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133349.log 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132645.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81228.log 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133349.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.132645.log.gz 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.98632.log 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81228.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.133349.log.gz 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81228.log.gz 2026-02-20T21:47:19.482 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64459.log 2026-02-20T21:47:19.483 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.98632.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88446.log 2026-02-20T21:47:19.483 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.98632.log.gz 2026-02-20T21:47:19.483 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.64459.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85280.log 2026-02-20T21:47:19.483 INFO:teuthology.orchestra.run.trial163.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.64459.log.gz 2026-02-20T21:47:19.483 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88446.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136497.log 2026-02-20T21:47:19.483 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85280.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.85280.log.gz 2026-02-20T21:47:19.483 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50972.log 2026-02-20T21:47:19.483 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136497.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136497.log.gz 2026-02-20T21:47:19.483 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88446.log.gz 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.107568.log 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.50972.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.138957.log 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50972.log.gz/var/log/ceph/cluster1-client.mirror.107568.log: 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74393.log 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.138957.log: 85.7% -- replaced with /var/log/ceph/cluster2-client.mirror.138957.log.gz 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69675.log 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.74393.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.107568.log.gz 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70826.log 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.69675.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.69675.log.gz 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.74393.log.gz 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.141820.log 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.108008.log 2026-02-20T21:47:19.494 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.70826.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70826.log.gz 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.141820.log: 88.3%gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60387.log 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.141820.log.gz 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.108008.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.108008.log.gz 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50692.log 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86760.log 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.60387.log: /var/log/ceph/cluster2-client.mirror.50692.log: 88.8% -- replaced with /var/log/ceph/cluster1-client.admin.60387.log.gz 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50692.log.gz 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58600.log 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.86760.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.133975.log 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86760.log.gz 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.58600.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81751.log 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.133975.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58600.log.gz 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.133975.log.gz 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130181.log 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77752.log 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81751.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81751.log.gz/var/log/ceph/cluster2-client.mirror.130181.log: 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.130181.log.gz 2026-02-20T21:47:19.495 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139964.log 2026-02-20T21:47:19.496 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.77752.log: gzip 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77752.log.gz 2026-02-20T21:47:19.496 INFO:teuthology.orchestra.run.trial163.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86189.log 2026-02-20T21:47:19.496 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122409.log 2026-02-20T21:47:19.496 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139964.log: /var/log/ceph/cluster2-client.mirror.86189.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.86189.log.gz 2026-02-20T21:47:19.497 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.123635.log 2026-02-20T21:47:19.497 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122409.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122409.log.gz 2026-02-20T21:47:19.497 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59085.log 2026-02-20T21:47:19.498 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.123635.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.123635.log.gz 2026-02-20T21:47:19.498 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67904.log 2026-02-20T21:47:19.499 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59085.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59085.log.gz 2026-02-20T21:47:19.499 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93211.log 2026-02-20T21:47:19.500 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67904.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67904.log.gz 2026-02-20T21:47:19.500 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107491.log 2026-02-20T21:47:19.501 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.93211.log: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93211.log.gz 2026-02-20T21:47:19.501 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88919.log 2026-02-20T21:47:19.502 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107491.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107491.log.gz 2026-02-20T21:47:19.502 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.122738.log 2026-02-20T21:47:19.505 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.88919.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88919.log.gz 2026-02-20T21:47:19.506 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134592.log 2026-02-20T21:47:19.506 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.122738.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.122738.log.gz 2026-02-20T21:47:19.507 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47683.log 2026-02-20T21:47:19.508 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134592.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.134592.log.gz 2026-02-20T21:47:19.508 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.99787.log 2026-02-20T21:47:19.509 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.47683.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47683.log.gz 2026-02-20T21:47:19.509 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132545.log 2026-02-20T21:47:19.510 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.99787.log: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.99787.log.gz 2026-02-20T21:47:19.511 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.45684.log 2026-02-20T21:47:19.512 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132545.log: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.132545.log.gz 2026-02-20T21:47:19.514 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.107410.log 2026-02-20T21:47:19.514 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.admin.45684.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.45684.log.gz 2026-02-20T21:47:19.514 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130955.log 2026-02-20T21:47:19.514 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.107410.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.107410.log.gz 2026-02-20T21:47:19.515 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75014.log 2026-02-20T21:47:19.524 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130955.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81565.log 2026-02-20T21:47:19.525 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.75014.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.75014.log.gz 2026-02-20T21:47:19.529 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.93993.log.gz 2026-02-20T21:47:19.530 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83849.log 2026-02-20T21:47:19.530 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81565.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81565.log.gz 2026-02-20T21:47:19.531 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66788.log 2026-02-20T21:47:19.534 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.83849.log: 93.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83849.log.gz 2026-02-20T21:47:19.534 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136453.log 2026-02-20T21:47:19.536 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.66788.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.66788.log.gz 2026-02-20T21:47:19.536 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101432.log 2026-02-20T21:47:19.536 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136453.log: 1.3% -- replaced with /var/log/ceph/cluster1-client.mirror.136453.log.gz 2026-02-20T21:47:19.536 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.120233.log 2026-02-20T21:47:19.537 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101432.log: 92.7% -- replaced with /var/log/ceph/cluster2-client.mirror.101432.log.gz 2026-02-20T21:47:19.538 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56757.log 2026-02-20T21:47:19.539 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.120233.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.120233.log.gz 2026-02-20T21:47:19.539 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68404.log 2026-02-20T21:47:19.541 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.56757.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56757.log.gz 2026-02-20T21:47:19.541 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.97657.log 2026-02-20T21:47:19.542 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.68404.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.68404.log.gz 2026-02-20T21:47:19.542 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132203.log 2026-02-20T21:47:19.544 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.97657.log: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.97657.log.gz 2026-02-20T21:47:19.544 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96329.log 2026-02-20T21:47:19.544 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132203.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.132203.log.gz 2026-02-20T21:47:19.544 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88770.log 2026-02-20T21:47:19.546 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.96329.log: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.96329.log.gz 2026-02-20T21:47:19.546 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-20T21:47:19.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88770.log: 95.5% -- replaced with /var/log/ceph/cluster2-client.mirror.102120.log.gz 2026-02-20T21:47:19.550 INFO:teuthology.orchestra.run.trial163.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88770.log.gz 2026-02-20T21:47:19.550 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.115923.log 2026-02-20T21:47:19.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81042.log 2026-02-20T21:47:19.550 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.115923.log: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.115923.log.gz 2026-02-20T21:47:19.570 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81286.log 2026-02-20T21:47:19.570 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81042.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81042.log.gz 2026-02-20T21:47:19.582 INFO:teuthology.orchestra.run.trial163.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.2.94146.log.gz 2026-02-20T21:47:19.582 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94645.log 2026-02-20T21:47:19.582 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88469.log 2026-02-20T21:47:19.582 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.81286.log: /var/log/ceph/cluster2-client.mirror.94645.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81286.log.gz 2026-02-20T21:47:19.582 INFO:teuthology.orchestra.run.trial163.stderr: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.94645.log.gz 2026-02-20T21:47:19.582 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85548.log 2026-02-20T21:47:19.583 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.88469.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.132716.log 2026-02-20T21:47:19.583 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.85548.log: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.85548.log.gz 2026-02-20T21:47:19.584 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.140950.log 2026-02-20T21:47:19.584 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.132716.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.132716.log.gz 2026-02-20T21:47:19.584 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59826.log 2026-02-20T21:47:19.584 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.140950.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.140950.log.gz 2026-02-20T21:47:19.586 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67997.log 2026-02-20T21:47:19.586 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.59826.log: 93.5% 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59826.log.gz 2026-02-20T21:47:19.586 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.88469.log.gz 2026-02-20T21:47:19.586 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72453.log 2026-02-20T21:47:19.586 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.101375.log 2026-02-20T21:47:19.587 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67997.log: /var/log/ceph/cluster2-client.mirror.72453.log: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.67997.log.gz 2026-02-20T21:47:19.587 INFO:teuthology.orchestra.run.trial163.stderr: 88.8% -- replaced with /var/log/ceph/cluster2-client.mirror.72453.log.gz 2026-02-20T21:47:19.587 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59050.log 2026-02-20T21:47:19.587 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.139578.log 2026-02-20T21:47:19.589 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.101375.log: /var/log/ceph/cluster2-client.mirror.59050.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.101375.log.gz 2026-02-20T21:47:19.589 INFO:teuthology.orchestra.run.trial163.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59050.log.gz 2026-02-20T21:47:19.589 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.127654.log 2026-02-20T21:47:19.589 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46726.log 2026-02-20T21:47:19.590 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.139578.log: /var/log/ceph/cluster2-client.mirror.127654.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.127654.log.gz 2026-02-20T21:47:19.590 INFO:teuthology.orchestra.run.trial163.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-client.mirror.139578.log.gz 2026-02-20T21:47:19.590 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67346.log 2026-02-20T21:47:19.590 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65482.log 2026-02-20T21:47:19.590 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.46726.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46726.log.gz 2026-02-20T21:47:19.591 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.67346.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.134705.log 2026-02-20T21:47:19.592 INFO:teuthology.orchestra.run.trial163.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.67346.log.gz 2026-02-20T21:47:19.592 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.admin.65482.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.admin.65482.log.gz 2026-02-20T21:47:19.592 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.102591.log 2026-02-20T21:47:19.592 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.130856.log 2026-02-20T21:47:19.592 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.134705.log: /var/log/ceph/cluster2-client.mirror.102591.log: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.102591.log.gz 2026-02-20T21:47:19.593 INFO:teuthology.orchestra.run.trial163.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.134705.log.gz 2026-02-20T21:47:19.593 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88942.log 2026-02-20T21:47:19.593 INFO:teuthology.orchestra.run.trial163.stderr: 95.6%gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.136947.log 2026-02-20T21:47:19.593 INFO:teuthology.orchestra.run.trial163.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.118628.log.gz 2026-02-20T21:47:19.594 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.130856.log: /var/log/ceph/cluster2-client.mirror.88942.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.130856.log.gz 2026-02-20T21:47:19.594 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80542.log 2026-02-20T21:47:19.595 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster1-client.mirror.136947.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.136947.log.gz 2026-02-20T21:47:19.595 INFO:teuthology.orchestra.run.trial163.stderr:/var/log/ceph/cluster2-client.mirror.80542.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80542.log.gz 2026-02-20T21:47:19.596 INFO:teuthology.orchestra.run.trial163.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.88942.log.gz 2026-02-20T21:47:19.605 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90937.log.gz 2026-02-20T21:47:19.617 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93457.log.gz 2026-02-20T21:47:19.641 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90060.log.gz 2026-02-20T21:47:19.727 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90238.log.gz 2026-02-20T21:47:19.785 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.116501.log.gz 2026-02-20T21:47:19.803 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.89783.log.gz 2026-02-20T21:47:19.914 INFO:teuthology.orchestra.run.trial163.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.130955.log.gz 2026-02-20T21:47:19.969 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.92511.log.gz 2026-02-20T21:47:20.298 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.76569.log.gz 2026-02-20T21:47:20.371 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.139964.log.gz 2026-02-20T21:47:20.391 INFO:teuthology.orchestra.run.trial163.stderr: 95.7% -- replaced with /var/log/ceph/cluster2-client.mirror.140814.log.gz 2026-02-20T21:47:20.556 INFO:teuthology.orchestra.run.trial163.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.1.76615.log.gz 2026-02-20T21:47:20.850 INFO:teuthology.orchestra.run.trial163.stderr: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.91693.log.gz 2026-02-20T21:47:21.209 INFO:teuthology.orchestra.run.trial163.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.49340.log.gz 2026-02-20T21:47:22.328 INFO:teuthology.orchestra.run.trial163.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.47140.log.gz 2026-02-20T21:47:23.098 INFO:teuthology.orchestra.run.trial163.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.3.53142.log.gz 2026-02-20T21:47:24.811 INFO:teuthology.orchestra.run.trial163.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.1.90632.log.gz 2026-02-20T21:47:26.937 INFO:teuthology.orchestra.run.trial163.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.84077.log.gz 2026-02-20T21:49:19.326 INFO:teuthology.orchestra.run.trial069.stderr: 93.7% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-20T21:49:23.833 INFO:teuthology.orchestra.run.trial069.stderr: 93.6% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-20T21:49:31.539 INFO:teuthology.orchestra.run.trial163.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-20T21:49:41.997 INFO:teuthology.orchestra.run.trial163.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-20T21:50:04.836 INFO:teuthology.orchestra.run.trial069.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-20T21:50:04.839 INFO:teuthology.orchestra.run.trial069.stderr: 2026-02-20T21:50:04.839 INFO:teuthology.orchestra.run.trial069.stderr:real 2m49.671s 2026-02-20T21:50:04.839 INFO:teuthology.orchestra.run.trial069.stderr:user 6m42.942s 2026-02-20T21:50:04.839 INFO:teuthology.orchestra.run.trial069.stderr:sys 0m19.519s 2026-02-20T21:50:42.775 INFO:teuthology.orchestra.run.trial163.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-20T21:50:42.778 INFO:teuthology.orchestra.run.trial163.stderr: 2026-02-20T21:50:42.778 INFO:teuthology.orchestra.run.trial163.stderr:real 3m27.583s 2026-02-20T21:50:42.778 INFO:teuthology.orchestra.run.trial163.stderr:user 9m34.636s 2026-02-20T21:50:42.778 INFO:teuthology.orchestra.run.trial163.stderr:sys 0m31.184s 2026-02-20T21:50:42.778 INFO:tasks.ceph:Archiving logs... 2026-02-20T21:50:42.779 DEBUG:teuthology.misc:Transferring archived files from trial069:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424/remote/trial069/log 2026-02-20T21:50:42.780 DEBUG:teuthology.orchestra.run.trial069:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-20T21:51:12.662 DEBUG:teuthology.misc:Transferring archived files from trial163:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424/remote/trial163/log 2026-02-20T21:51:12.663 DEBUG:teuthology.orchestra.run.trial163:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-20T21:51:52.577 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-20T21:51:52.587 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-20T21:51:52.587 DEBUG:teuthology.orchestra.run.trial069:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-20T21:51:52.590 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-20T21:51:52.741 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-20T21:51:52.752 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-20T21:51:52.752 DEBUG:teuthology.orchestra.run.trial069:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-20T21:51:52.754 DEBUG:teuthology.orchestra.run.trial163:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-20T21:51:52.768 INFO:teuthology.orchestra.run.trial069.stderr:bash: line 1: ntpq: command not found 2026-02-20T21:51:52.771 INFO:teuthology.orchestra.run.trial163.stderr:bash: line 1: ntpq: command not found 2026-02-20T21:51:52.963 INFO:teuthology.orchestra.run.trial069.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-20T21:51:52.963 INFO:teuthology.orchestra.run.trial069.stdout:=============================================================================== 2026-02-20T21:51:52.963 INFO:teuthology.orchestra.run.trial069.stdout:^* ntp4.glypnod.com 2 8 377 244 +107us[ +355us] +/- 15ms 2026-02-20T21:51:52.963 INFO:teuthology.orchestra.run.trial069.stdout:^- 23.186.168.130 2 7 177 116 +224us[ +224us] +/- 56ms 2026-02-20T21:51:52.963 INFO:teuthology.orchestra.run.trial069.stdout:^- ntp2.zeus.frumentum.media 2 7 377 118 -106ms[ -106ms] +/- 129ms 2026-02-20T21:51:52.963 INFO:teuthology.orchestra.run.trial069.stdout:^- ntp6.kernfusion.at 2 8 377 250 +109us[ +356us] +/- 47ms 2026-02-20T21:51:52.963 INFO:teuthology.orchestra.run.trial163.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-20T21:51:52.963 INFO:teuthology.orchestra.run.trial163.stdout:=============================================================================== 2026-02-20T21:51:52.963 INFO:teuthology.orchestra.run.trial163.stdout:^* ntp4.glypnod.com 2 8 377 318 -1436us[-1381us] +/- 17ms 2026-02-20T21:51:52.963 INFO:teuthology.orchestra.run.trial163.stdout:^- 23.186.168.130 2 7 277 123 +361us[ +361us] +/- 56ms 2026-02-20T21:51:52.963 INFO:teuthology.orchestra.run.trial163.stdout:^- ntp2.zeus.frumentum.media 2 6 377 57 -106ms[ -106ms] +/- 129ms 2026-02-20T21:51:52.963 INFO:teuthology.orchestra.run.trial163.stdout:^- ntp6.kernfusion.at 2 7 377 50 +184us[ +184us] +/- 47ms 2026-02-20T21:51:52.964 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-20T21:51:52.977 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-20T21:51:52.977 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-20T21:51:52.993 DEBUG:teuthology.orchestra.run.trial069:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-20T21:51:53.059 DEBUG:teuthology.orchestra.run.trial163:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-20T21:51:53.117 DEBUG:teuthology.orchestra.run.trial069:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2026-02-20T21:51:53.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:51:53.145 DEBUG:teuthology.orchestra.run.trial163:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 2026-02-20T21:51:53.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:51:53.172 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-20T21:51:53.192 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-20T21:51:53.211 INFO:teuthology.task.internal:Duration was 3069.883792 seconds 2026-02-20T21:51:53.212 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-20T21:51:53.232 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-20T21:51:53.233 DEBUG:teuthology.orchestra.run.trial069:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-20T21:51:53.236 DEBUG:teuthology.orchestra.run.trial163:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-20T21:51:53.271 INFO:teuthology.orchestra.run.trial069.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-20T21:51:53.277 INFO:teuthology.orchestra.run.trial163.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-20T21:51:53.565 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-20T21:51:53.566 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial069.front.sepia.ceph.com 2026-02-20T21:51:53.566 DEBUG:teuthology.orchestra.run.trial069:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-20T21:51:53.589 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial163.front.sepia.ceph.com 2026-02-20T21:51:53.589 DEBUG:teuthology.orchestra.run.trial163:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-20T21:51:53.624 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-20T21:51:53.624 DEBUG:teuthology.orchestra.run.trial069:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-20T21:51:53.632 DEBUG:teuthology.orchestra.run.trial163:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-20T21:51:53.729 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-20T21:51:53.729 DEBUG:teuthology.orchestra.run.trial069:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-20T21:51:53.732 DEBUG:teuthology.orchestra.run.trial163:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-20T21:51:53.753 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T21:51:53.753 INFO:teuthology.orchestra.run.trial069.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T21:51:53.754 INFO:teuthology.orchestra.run.trial069.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-20T21:51:53.754 INFO:teuthology.orchestra.run.trial069.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-20T21:51:53.754 INFO:teuthology.orchestra.run.trial069.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-20T21:51:53.755 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-20T21:51:53.757 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-20T21:51:53.757 INFO:teuthology.orchestra.run.trial163.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-20T21:51:53.757 INFO:teuthology.orchestra.run.trial163.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-20T21:51:53.757 INFO:teuthology.orchestra.run.trial163.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-20T21:51:53.764 INFO:teuthology.orchestra.run.trial069.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 82.8% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-20T21:51:53.767 INFO:teuthology.orchestra.run.trial163.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 83.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-20T21:51:53.769 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-20T21:51:53.790 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-20T21:51:53.790 DEBUG:teuthology.orchestra.run.trial069:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-20T21:51:53.832 DEBUG:teuthology.orchestra.run.trial163:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-20T21:51:53.857 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-20T21:51:53.878 DEBUG:teuthology.orchestra.run.trial069:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-20T21:51:53.881 DEBUG:teuthology.orchestra.run.trial163:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-20T21:51:53.902 INFO:teuthology.orchestra.run.trial069.stdout:kernel.core_pattern = core 2026-02-20T21:51:53.921 INFO:teuthology.orchestra.run.trial163.stdout:kernel.core_pattern = core 2026-02-20T21:51:53.952 DEBUG:teuthology.orchestra.run.trial069:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-20T21:51:53.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:51:53.975 DEBUG:teuthology.orchestra.run.trial163:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-20T21:51:54.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-20T21:51:54.010 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-20T21:51:54.033 INFO:teuthology.task.internal:Transferring archived files... 2026-02-20T21:51:54.034 DEBUG:teuthology.misc:Transferring archived files from trial069:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424/remote/trial069 2026-02-20T21:51:54.034 DEBUG:teuthology.orchestra.run.trial069:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-20T21:51:54.072 DEBUG:teuthology.misc:Transferring archived files from trial163:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-20_16:15:01-rbd-wip-pdonnell-testing-20260219.182737-tentacle-distro-default-trial/61424/remote/trial163 2026-02-20T21:51:54.072 DEBUG:teuthology.orchestra.run.trial163:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-20T21:51:54.110 INFO:teuthology.task.internal:Removing archive directory... 2026-02-20T21:51:54.111 DEBUG:teuthology.orchestra.run.trial069:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-20T21:51:54.113 DEBUG:teuthology.orchestra.run.trial163:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-20T21:51:54.159 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-20T21:51:54.182 INFO:teuthology.task.internal:Not uploading archives. 2026-02-20T21:51:54.182 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-20T21:51:54.204 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-20T21:51:54.204 DEBUG:teuthology.orchestra.run.trial069:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-20T21:51:54.207 DEBUG:teuthology.orchestra.run.trial163:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-20T21:51:54.221 INFO:teuthology.orchestra.run.trial069.stdout: 917572 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 20 21:51 /home/ubuntu/cephtest 2026-02-20T21:51:54.225 INFO:teuthology.orchestra.run.trial163.stdout: 917521 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 20 21:51 /home/ubuntu/cephtest 2026-02-20T21:51:54.226 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-20T21:51:54.248 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-20T21:51:54.293 INFO:teuthology.run:Summary data: description: rbd/mirror-thrash/{base/install clients/mirror cluster/{2-node} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid policy/simple rbd-mirror/four-per-cluster supported-random-distro$/{centos_latest} workloads/rbd-mirror-journal-stress-workunit} duration: 3069.8837921619415 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-20T21:51:54.293 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-20T21:51:54.409 INFO:teuthology.run:pass